From 8a8a37b18743b71c98b1cc5f60707918130cf0a8 Mon Sep 17 00:00:00 2001 From: tharun-n Date: Thu, 28 Mar 2024 16:38:10 +0530 Subject: [PATCH 001/159] feature: added create feature route --- .vscode/settings.json | 4 ++-- core/api.py | 29 +++++++++++++++++++++++++++++ core/schemas.py | 2 ++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4d8aaca..ffdc12d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,8 +16,8 @@ "[python]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports": true, - "source.fixAll": true + "source.organizeImports": "explicit", + "source.fixAll": "explicit" } }, diff --git a/core/api.py b/core/api.py index 0a3a409..904e4ee 100644 --- a/core/api.py +++ b/core/api.py @@ -36,6 +36,7 @@ SyncSchemaUpdateIn, SyncStartStopSchemaIn, UserSchemaOut, + CreateConfigSchemaIn ) from .models import Account, Connector, Credential, Destination, Source, Sync, User, Workspace, OAuthApiKeys @@ -144,6 +145,34 @@ def connector_discover(request, workspace_id, connector_type, payload: Connector ).json() +@router.post("/workspaces/{workspace_id}/connectors/{connector_type}/create", response=Dict) +def connector_create(request, workspace_id, connector_type, payload: CreateConfigSchemaIn): + workspace = Workspace.objects.get(id=workspace_id) + connector = Connector.objects.get(type=connector_type) + queryset = OAuthApiKeys.objects.filter(workspace=workspace, type=connector_type) + if queryset.exists(): + keys = queryset.first() + + logger.debug("connector spec keys:-", keys) + # Replacing oauth keys with db values + payload.config = replace_values_in_json(payload.config, keys.oauth_config) + + else: + # Replacing oauth keys with .env values + oauth_proxy_keys = config("OAUTH_SECRETS", default="", cast=Csv(str)) + if len(oauth_proxy_keys) > 0: + config_str = json.dumps(payload.config) + for key in oauth_proxy_keys: + config_str = config_str.replace(key, config(key)) + payload.config = json.loads(config_str) + + res = requests.post( + f"{CONNECTOR_PREFIX_URL}/{connector.type}/create", + json={**payload.dict(), "docker_image": connector.docker_image, "docker_tag": connector.docker_tag}, + timeout=SHORT_TIMEOUT, + ).json() + return res + @router.get("/workspaces/{workspace_id}/credentials/", response=List[CredentialSchema]) def list_credentials(request, workspace_id): workspace = Workspace.objects.get(id=workspace_id) diff --git a/core/schemas.py b/core/schemas.py index 326c9f4..59c1f9b 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -59,6 +59,8 @@ class Config(CamelSchemaConfig): class ConnectorConfigSchemaIn(Schema): config: Dict +class CreateConfigSchemaIn(Schema): + config: Dict class SyncStartStopSchemaIn(Schema): full_refresh: bool = False From 58ad45c9eb62175fddf98181af2d63753bce2fee Mon Sep 17 00:00:00 2001 From: Nagendra Date: Tue, 2 Apr 2024 17:05:29 +0530 Subject: [PATCH 002/159] [Story]: updated connector defs & added oauth_schema for shopify --- core/api.py | 4 + core/migrations/0013_connector_mode.py | 18 ++++ core/migrations/0014_auto_20240402_1125.py | 19 ++++ core/migrations/0015_auto_20240402_1128.py | 19 ++++ core/models.py | 6 ++ core/oauth_schema.json | 22 ++++ core/schemas.py | 4 +- init_db/connector_def.json | 111 ++------------------- init_db/connector_init.py | 1 + 9 files changed, 99 insertions(+), 105 deletions(-) create mode 100644 core/migrations/0013_connector_mode.py create mode 100644 core/migrations/0014_auto_20240402_1125.py create mode 100644 core/migrations/0015_auto_20240402_1128.py diff --git a/core/api.py b/core/api.py index 904e4ee..0c77537 100644 --- a/core/api.py +++ b/core/api.py @@ -173,6 +173,7 @@ def connector_create(request, workspace_id, connector_type, payload: CreateConfi ).json() return res + @router.get("/workspaces/{workspace_id}/credentials/", response=List[CredentialSchema]) def list_credentials(request, workspace_id): workspace = Workspace.objects.get(id=workspace_id) @@ -422,10 +423,13 @@ def get_connectors(request): try: logger.debug("listing connectors") connectors = Connector.objects.all() + + logger.info(f"connectors - {connectors}") src_dst_dict: Dict[str, List[ConnectorSchema]] = {} src_dst_dict["SRC"] = [] src_dst_dict["DEST"] = [] for conn in connectors: + logger.info(f"conn{conn}") arr = conn.type.split("_") if arr[0] == "SRC": src_dst_dict["SRC"].append(conn) diff --git a/core/migrations/0013_connector_mode.py b/core/migrations/0013_connector_mode.py new file mode 100644 index 0000000..ea3c7aa --- /dev/null +++ b/core/migrations/0013_connector_mode.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-04-02 10:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_auto_20240212_0639'), + ] + + operations = [ + migrations.AddField( + model_name='connector', + name='mode', + field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), + ), + ] diff --git a/core/migrations/0014_auto_20240402_1125.py b/core/migrations/0014_auto_20240402_1125.py new file mode 100644 index 0000000..f686bb1 --- /dev/null +++ b/core/migrations/0014_auto_20240402_1125.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.5 on 2024-04-02 11:25 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_connector_mode'), + ] + + operations = [ + migrations.AlterField( + model_name='connector', + name='mode', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), size=None), + ), + ] diff --git a/core/migrations/0015_auto_20240402_1128.py b/core/migrations/0015_auto_20240402_1128.py new file mode 100644 index 0000000..7fd7cc1 --- /dev/null +++ b/core/migrations/0015_auto_20240402_1128.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.5 on 2024-04-02 11:28 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0014_auto_20240402_1125'), + ] + + operations = [ + migrations.AlterField( + model_name='connector', + name='mode', + field=django.contrib.postgres.fields.ArrayField(base_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), size=None), size=None), + ), + ] diff --git a/core/models.py b/core/models.py index 768f377..6371eb8 100644 --- a/core/models.py +++ b/core/models.py @@ -11,6 +11,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import AbstractUser from django.db import models +from django.contrib.postgres.fields import ArrayField from enum import Enum @@ -128,6 +129,11 @@ class Connector(models.Model): status = models.CharField(max_length=256, null=False, blank=False, default="active") oauth = models.BooleanField(default=False) oauth_keys = models.CharField(max_length=64, choices=OAuthKeys.choices(), default=OAuthKeys.PRIVATE.value) + mode = ArrayField( + ArrayField( + models.CharField(max_length=64, null=False, blank=False, default="DUMMY_CONNECTOR_MODE") + ) + ) class Account(models.Model): diff --git a/core/oauth_schema.json b/core/oauth_schema.json index 055441e..549e187 100644 --- a/core/oauth_schema.json +++ b/core/oauth_schema.json @@ -117,6 +117,28 @@ ], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#" + }, + "SRC_SHOPIFY": { + "type": "object", + "properties": { + "AUTH_SHOPIFY_CLIENT_ID": { + "type": "string", + "description": "Shopify client id", + "title":"Client Id" + }, + "AUTH_SHOPIFY_CLIENT_SECRET": { + "type": "string", + "description": "Shopify client secret", + "title":"Client Secret" + + } + }, + "required": [ + "AUTH_SHOPIFY_CLIENT_ID", + "AUTH_SHOPIFY_CLIENT_SECRET" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" } } } \ No newline at end of file diff --git a/core/schemas.py b/core/schemas.py index 59c1f9b..5e25159 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -53,15 +53,17 @@ class Config(CamelSchemaConfig): class ConnectorSchema(ModelSchema): class Config(CamelSchemaConfig): model = Connector - model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys"] + model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] class ConnectorConfigSchemaIn(Schema): config: Dict + class CreateConfigSchemaIn(Schema): config: Dict + class SyncStartStopSchemaIn(Schema): full_refresh: bool = False diff --git a/init_db/connector_def.json b/init_db/connector_def.json index e4baf24..f861fc2 100644 --- a/init_db/connector_def.json +++ b/init_db/connector_def.json @@ -2,66 +2,13 @@ "definitions": [ { "type": "SRC", - "unique_name": "POSTGRES", - "docker_image": "valmiio/source-postgres", + "unique_name": "SHOPIFY", + "docker_image": "valmiio/source-shopify", "docker_tag": "latest", - "display_name": "Postgres", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "SRC", - "unique_name": "REDSHIFT", - "docker_image": "valmiio/source-redshift", - "docker_tag": "latest", - "display_name": "Redshift", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "SRC", - "unique_name": "SNOWFLAKE", - "docker_image": "valmiio/source-snowflake", - "docker_tag": "latest", - "display_name": "Snowflake", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "WEBHOOK", - "docker_image": "valmiio/destination-webhook", - "docker_tag": "latest", - "display_name": "Webhook", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "GOOGLE-SHEETS", - "docker_image": "valmiio/destination-google-sheets", - "docker_tag": "latest", - "display_name": "Google Sheets", - "oauth": "True", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "GOOGLE-ADS", - "docker_image": "valmiio/destination-google-ads", - "docker_tag": "latest", - "display_name": "Google Ads", + "display_name": "Shopify", "oauth": "True", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "CUSTOMER-IO", - "docker_image": "valmiio/destination-customer-io", - "docker_tag": "latest", - "display_name": "Customer.io", - "oauth": "False", - "oauth_keys":"private" + "oauth_keys":"private", + "mode":["etl"] }, { "type": "DEST", @@ -70,52 +17,8 @@ "docker_tag": "latest", "display_name": "Facebook Ads", "oauth": "True", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "SLACK", - "docker_image": "valmiio/destination-slack", - "docker_tag": "latest", - "display_name": "Slack", - "oauth": "True", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "HUBSPOT", - "docker_image": "valmiio/destination-hubspot", - "docker_tag": "latest", - "display_name": "Hubspot", - "oauth": "True", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "Stripe", - "docker_image": "valmiio/destination-stripe", - "docker_tag": "latest", - "display_name": "Stripe", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "Gong", - "docker_image": "valmiio/destination-gong", - "docker_tag": "latest", - "display_name": "Gong", - "oauth": "False", - "oauth_keys":"private" - }, - { - "type": "DEST", - "unique_name": "ANDROID-PUSH-NOTIFICATIONS", - "docker_image": "valmiio/destination-android-push-notifications", - "docker_tag": "latest", - "display_name": "Android Push Notifications", - "oauth": "False", - "oauth_keys":"private" + "oauth_keys":"private", + "mode":["etl"] } ] } diff --git a/init_db/connector_init.py b/init_db/connector_init.py index 254b08f..e349d5f 100755 --- a/init_db/connector_init.py +++ b/init_db/connector_init.py @@ -24,6 +24,7 @@ "docker_tag": conn_def["docker_tag"], "oauth": conn_def["oauth"], "oauth_keys": conn_def["oauth_keys"], + "mode": conn_def["mode"], }, auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), ) From bd1e97b8ae7f32aa55f63582b224f3d3c65d84ce Mon Sep 17 00:00:00 2001 From: Nagendra Date: Tue, 2 Apr 2024 17:44:51 +0530 Subject: [PATCH 003/159] [Story]:fix -default array field error in connector model --- core/migrations/0013_connector_mode.py | 6 ++++-- core/migrations/0014_auto_20240402_1125.py | 19 ------------------- core/migrations/0015_auto_20240402_1128.py | 19 ------------------- core/models.py | 6 +----- 4 files changed, 5 insertions(+), 45 deletions(-) delete mode 100644 core/migrations/0014_auto_20240402_1125.py delete mode 100644 core/migrations/0015_auto_20240402_1128.py diff --git a/core/migrations/0013_connector_mode.py b/core/migrations/0013_connector_mode.py index ea3c7aa..6019eb2 100644 --- a/core/migrations/0013_connector_mode.py +++ b/core/migrations/0013_connector_mode.py @@ -1,5 +1,6 @@ -# Generated by Django 3.1.5 on 2024-04-02 10:43 +# Generated by Django 3.1.5 on 2024-04-02 12:07 +import django.contrib.postgres.fields from django.db import migrations, models @@ -13,6 +14,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='connector', name='mode', - field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), blank=True, default=list, size=None), ), ] diff --git a/core/migrations/0014_auto_20240402_1125.py b/core/migrations/0014_auto_20240402_1125.py deleted file mode 100644 index f686bb1..0000000 --- a/core/migrations/0014_auto_20240402_1125.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.1.5 on 2024-04-02 11:25 - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0013_connector_mode'), - ] - - operations = [ - migrations.AlterField( - model_name='connector', - name='mode', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), size=None), - ), - ] diff --git a/core/migrations/0015_auto_20240402_1128.py b/core/migrations/0015_auto_20240402_1128.py deleted file mode 100644 index 7fd7cc1..0000000 --- a/core/migrations/0015_auto_20240402_1128.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.1.5 on 2024-04-02 11:28 - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0014_auto_20240402_1125'), - ] - - operations = [ - migrations.AlterField( - model_name='connector', - name='mode', - field=django.contrib.postgres.fields.ArrayField(base_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='DUMMY_CONNECTOR_MODE', max_length=64), size=None), size=None), - ), - ] diff --git a/core/models.py b/core/models.py index 6371eb8..bbb219f 100644 --- a/core/models.py +++ b/core/models.py @@ -129,11 +129,7 @@ class Connector(models.Model): status = models.CharField(max_length=256, null=False, blank=False, default="active") oauth = models.BooleanField(default=False) oauth_keys = models.CharField(max_length=64, choices=OAuthKeys.choices(), default=OAuthKeys.PRIVATE.value) - mode = ArrayField( - ArrayField( - models.CharField(max_length=64, null=False, blank=False, default="DUMMY_CONNECTOR_MODE") - ) - ) + mode = ArrayField(models.CharField(max_length=64), blank=True, default=list) class Account(models.Model): From 0268de20fbcd200cfd10d09972ed6e042d4d5caf Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 12:13:18 +0530 Subject: [PATCH 004/159] added storage credentials --- core/api.py | 39 ++++++++++++++++++++-- core/migrations/0013_storagecredentials.py | 23 +++++++++++++ core/models.py | 5 +++ init_db/connector_def.json | 10 ++++++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 core/migrations/0013_storagecredentials.py diff --git a/core/api.py b/core/api.py index 0c77537..e47911c 100644 --- a/core/api.py +++ b/core/api.py @@ -9,6 +9,12 @@ import json import logging import uuid +import psycopg2 +import uuid +import os +import string +import random + from datetime import datetime from typing import Dict, List, Optional @@ -39,7 +45,7 @@ CreateConfigSchemaIn ) -from .models import Account, Connector, Credential, Destination, Source, Sync, User, Workspace, OAuthApiKeys +from .models import Account, Connector, Credential, Destination, Source, StorageCredentials, Sync, User, Workspace, OAuthApiKeys from valmi_app_backend.utils import replace_values_in_json @@ -186,6 +192,35 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): data = payload.dict() try: logger.debug(data) + logger.debug(workspace_id) + source = data.get("connector_type") + if source=="SRC_SHOPIFY": + host_url = os.environ["DB_URL"] + db_password = os.environ["DB_PASSWORD"] + db_username = os.environ["DB_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + create_new_cred = True + try: + do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) + create_new_cred = False + except Exception: + create_new_cred = True + if create_new_cred==True: + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace_id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + logger.debug(result) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) data["connector"] = Connector.objects.get(type=data.pop("connector_type")) @@ -211,7 +246,6 @@ def update_credential(request, workspace_id, payload: CredentialSchemaUpdateIn): credential = Credential.objects.filter(id=data.pop("id")) data["workspace"] = Workspace.objects.get(id=workspace_id) data["connector"] = Connector.objects.get(type=data.pop("connector_type")) - account_info = data.pop("account", None) if account_info and len(account_info) > 0: account = Account.objects.filter(id=account_info.pop("id")) @@ -238,6 +272,7 @@ def list_sources(request, workspace_id): @router.post("/workspaces/{workspace_id}/sources/create", response={200: SourceSchema, 400: DetailSchema}) def create_source(request, workspace_id, payload: SourceSchemaIn): data = payload.dict() + logger.debug("Creating source") logger.debug(dict) try: data["id"] = uuid.uuid4() diff --git a/core/migrations/0013_storagecredentials.py b/core/migrations/0013_storagecredentials.py new file mode 100644 index 0000000..eb1bf46 --- /dev/null +++ b/core/migrations/0013_storagecredentials.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.5 on 2024-04-02 08:20 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_auto_20240212_0639'), + ] + + operations = [ + migrations.CreateModel( + name='StorageCredentials', + fields=[ + ('id', models.UUIDField(default=uuid.UUID('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'), editable=False, primary_key=True, serialize=False)), + ('connector_config', models.JSONField()), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='storage_credentials', to='core.workspace')), + ], + ), + ] diff --git a/core/models.py b/core/models.py index bbb219f..d720d70 100644 --- a/core/models.py +++ b/core/models.py @@ -82,7 +82,12 @@ class Credential(models.Model): def __str__(self): return f"{self.connector}: {self.connector_config} : {self.workspace}: {self.id} : {self.name}" + +class StorageCredentials(models.Model): + id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) + workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="storage_credentials") + connector_config = models.JSONField(blank=False, null=False) class Source(models.Model): created_at = models.DateTimeField(auto_now_add=True) diff --git a/init_db/connector_def.json b/init_db/connector_def.json index f861fc2..9095e33 100644 --- a/init_db/connector_def.json +++ b/init_db/connector_def.json @@ -19,6 +19,16 @@ "oauth": "True", "oauth_keys":"private", "mode":["etl"] + }, + { + "type": "DEST", + "unique_name": "POSTGRES-DEST", + "docker_image": "valmiio/destination-postgres", + "docker_tag": "latest", + "display_name": "Dest Postgres", + "oauth": "False", + "oauth_keys":"private", + "mode":["etl"] } ] } From 52f2a23cf2103660ce28d8d27401db70d7d31367 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 12:15:12 +0530 Subject: [PATCH 005/159] migration changes --- core/migrations/0013_storagecredentials.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/migrations/0013_storagecredentials.py b/core/migrations/0013_storagecredentials.py index eb1bf46..3aede12 100644 --- a/core/migrations/0013_storagecredentials.py +++ b/core/migrations/0013_storagecredentials.py @@ -20,4 +20,10 @@ class Migration(migrations.Migration): ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='storage_credentials', to='core.workspace')), ], ), + migrations.AddField( + model_name='connector', + name='mode', + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), blank=True, default=list, size=None), + ), ] From 93f6082c85f536b80db12b8655d0e8b55493aec5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 16:44:10 +0530 Subject: [PATCH 006/159] get storage credentails and schema for destination --- core/api.py | 103 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/core/api.py b/core/api.py index e47911c..4129de6 100644 --- a/core/api.py +++ b/core/api.py @@ -8,12 +8,12 @@ import json import logging +import os +import random +import string import uuid import psycopg2 import uuid -import os -import string -import random from datetime import datetime from typing import Dict, List, Optional @@ -194,33 +194,33 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): logger.debug(data) logger.debug(workspace_id) source = data.get("connector_type") - if source=="SRC_SHOPIFY": - host_url = os.environ["DB_URL"] - db_password = os.environ["DB_PASSWORD"] - db_username = os.environ["DB_USERNAME"] - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - create_new_cred = True - try: - do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) - create_new_cred = False - except Exception: - create_new_cred = True - if create_new_cred==True: - user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password} - credential_info = {"id": uuid.uuid4()} - credential_info["workspace"] = Workspace.objects.get(id=workspace_id) - credential_info["connector_config"] = creds - result = StorageCredentials.objects.create(**credential_info) - logger.debug(result) - query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - cursor.execute(query, (password,)) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) - cursor.execute(query) - conn.commit() - conn.close() + # if source=="SRC_SHOPIFY": + # host_url = os.environ["DB_URL"] + # db_password = os.environ["DB_PASSWORD"] + # db_username = os.environ["DB_USERNAME"] + # conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + # cursor = conn.cursor() + # create_new_cred = True + # try: + # do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) + # create_new_cred = False + # except Exception: + # create_new_cred = True + # if create_new_cred: + # user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + # password = ''.join(random.choices(string.ascii_uppercase, k=17)) + # creds = {'username': user_name, 'password': password} + # credential_info = {"id": uuid.uuid4()} + # credential_info["workspace"] = Workspace.objects.get(id=workspace_id) + # credential_info["connector_config"] = creds + # result = StorageCredentials.objects.create(**credential_info) + # logger.debug(result) + # query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + # cursor.execute(query, (password,)) + # query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) + # cursor.execute(query) + # conn.commit() + # conn.close() data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) data["connector"] = Connector.objects.get(type=data.pop("connector_type")) @@ -236,6 +236,49 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): except Exception: logger.exception("Credential error") return {"detail": "The specific credential cannot be created."} + + +@router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) +def get_storage_credentials(request, workspace_id): + host_url = os.environ["DB_URL"] + db_password = os.environ["DB_PASSWORD"] + db_username = os.environ["DB_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + create_new_cred = True + try: + do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) + create_new_cred = False + except Exception: + create_new_cred = True + if create_new_cred: + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace_id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + logger.debug(result) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() + config = {} + if not create_new_cred: + config['username'] = do_id_exists.connector_config.get('username') + config['password'] = do_id_exists.connector_config.get('password') + else: + config['username'] = user_name + config['password'] = password + config['database'] = "dvdrental" + config['host'] = host_url + config['port'] = 5432 + config["ssl"] = False + config["schema"] = "public" + return json.dumps(config) @router.post("/workspaces/{workspace_id}/credentials/update", response={200: CredentialSchema, 400: DetailSchema}) From f72e8273b7d941e599a65255e137c1913e0cdf09 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 18:03:33 +0530 Subject: [PATCH 007/159] schema creation --- core/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/api.py b/core/api.py index 4129de6..ec41e25 100644 --- a/core/api.py +++ b/core/api.py @@ -262,6 +262,8 @@ def get_storage_credentials(request, workspace_id): logger.debug(result) query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) cursor.execute(query, (password,)) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + cursor.execute(query) query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) cursor.execute(query) conn.commit() From 0b9b168f2c68fdcebd9e2f26d8aca46a8867beb1 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 18:07:31 +0530 Subject: [PATCH 008/159] schema creation --- core/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api.py b/core/api.py index ec41e25..be02ce8 100644 --- a/core/api.py +++ b/core/api.py @@ -279,7 +279,7 @@ def get_storage_credentials(request, workspace_id): config['host'] = host_url config['port'] = 5432 config["ssl"] = False - config["schema"] = "public" + config["schema"] = user_name return json.dumps(config) From 27454ee1111d56e1c0237a82111a9abeef82730f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 18:08:49 +0530 Subject: [PATCH 009/159] schema changes --- core/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/api.py b/core/api.py index be02ce8..215acc6 100644 --- a/core/api.py +++ b/core/api.py @@ -272,14 +272,15 @@ def get_storage_credentials(request, workspace_id): if not create_new_cred: config['username'] = do_id_exists.connector_config.get('username') config['password'] = do_id_exists.connector_config.get('password') + config["schema"] = do_id_exists.connector_config.get("username") else: config['username'] = user_name config['password'] = password + config["schema"] = user_name config['database'] = "dvdrental" config['host'] = host_url config['port'] = 5432 config["ssl"] = False - config["schema"] = user_name return json.dumps(config) From aaaf2095b76ac0d5ea03c43879822bb7253bb83f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 8 Apr 2024 18:17:42 +0530 Subject: [PATCH 010/159] updated schema permissions --- core/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api.py b/core/api.py index 215acc6..2c09de2 100644 --- a/core/api.py +++ b/core/api.py @@ -264,7 +264,7 @@ def get_storage_credentials(request, workspace_id): cursor.execute(query, (password,)) query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) cursor.execute(query) conn.commit() conn.close() From 97eb5d3fae69b0e84a7c87e690f9f5be4802b149 Mon Sep 17 00:00:00 2001 From: Nagendra Date: Mon, 8 Apr 2024 18:59:08 +0530 Subject: [PATCH 011/159] [Story]: added DB credentials in .env --- .env-example | 7 ++++++- ...toragecredentials.py => 0014_storagecredentials.py} | 10 ++-------- 2 files changed, 8 insertions(+), 9 deletions(-) rename core/migrations/{0013_storagecredentials.py => 0014_storagecredentials.py} (65%) diff --git a/.env-example b/.env-example index d2f32bc..b242bc1 100644 --- a/.env-example +++ b/.env-example @@ -41,4 +41,9 @@ OTEL_PYTHON_LOG_LEVEL="debug" OTEL_PYTHON_LOG_CORRELATION=True OTEL_EXPORTER_OTLP_INSECURE=True -#STREAM_API_URL="http://localhost:3100" \ No newline at end of file +#STREAM_API_URL="http://localhost:3100" + + +DB_URL="**********" +DB_USERNAME="*******" +DB_PASSWORD="*********" \ No newline at end of file diff --git a/core/migrations/0013_storagecredentials.py b/core/migrations/0014_storagecredentials.py similarity index 65% rename from core/migrations/0013_storagecredentials.py rename to core/migrations/0014_storagecredentials.py index 3aede12..d750c4d 100644 --- a/core/migrations/0013_storagecredentials.py +++ b/core/migrations/0014_storagecredentials.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.5 on 2024-04-02 08:20 +# Generated by Django 3.1.5 on 2024-04-08 12:58 from django.db import migrations, models import django.db.models.deletion @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ('core', '0012_auto_20240212_0639'), + ('core', '0013_connector_mode'), ] operations = [ @@ -20,10 +20,4 @@ class Migration(migrations.Migration): ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='storage_credentials', to='core.workspace')), ], ), - migrations.AddField( - model_name='connector', - name='mode', - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField(max_length=64), blank=True, default=list, size=None), - ), ] From 0f4942900294ab647a2c7fff39425f4b6184af88 Mon Sep 17 00:00:00 2001 From: Nagendra Date: Fri, 12 Apr 2024 15:39:46 +0530 Subject: [PATCH 012/159] [Story]: Included "mode" field within Credential Schema in the "/syncs" response --- core/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/schemas.py b/core/schemas.py index 5e25159..45584d3 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -100,6 +100,7 @@ class Config(CamelSchemaConfig): display_name: str = Field(None, alias="connector.display_name") oauth: str = Field(None, alias="connector.oauth") oauth_keys: str = Field(None, alias="connector.oauth_keys") + mode: list = Field(None, alias="connector.mode") account: AccountSchema = Field(None, alias="account") From fd5314c7f911760bfaf5f033a345ce979fbb55d9 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 12 Apr 2024 18:37:22 +0530 Subject: [PATCH 013/159] added prompts --- .env-example | 4 ++- core/api.py | 27 -------------- core/engine_api.py | 15 +++++++- core/migrations/0013_connector_mode.py | 20 ----------- core/migrations/0014_prompt.py | 21 +++++++++++ core/migrations/0015_auto_20240411_1531.py | 26 ++++++++++++++ core/migrations/0016_auto_20240412_0723.py | 41 ++++++++++++++++++++++ core/migrations/0017_auto_20240412_0730.py | 18 ++++++++++ core/migrations/0018_auto_20240412_1025.py | 21 +++++++++++ core/migrations/0019_auto_20240412_1228.py | 40 +++++++++++++++++++++ core/models.py | 12 +++++++ core/prompt_api.py | 26 ++++++++++++++ core/schemas.py | 13 ++++++- docker-entrypoint.sh | 1 + init_db/prompt_def.json | 13 +++++++ init_db/prompt_init.py | 29 +++++++++++++++ valmi_app_backend/urls.py | 2 ++ 17 files changed, 279 insertions(+), 50 deletions(-) delete mode 100644 core/migrations/0013_connector_mode.py create mode 100644 core/migrations/0014_prompt.py create mode 100644 core/migrations/0015_auto_20240411_1531.py create mode 100644 core/migrations/0016_auto_20240412_0723.py create mode 100644 core/migrations/0017_auto_20240412_0730.py create mode 100644 core/migrations/0018_auto_20240412_1025.py create mode 100644 core/migrations/0019_auto_20240412_1228.py create mode 100644 core/prompt_api.py create mode 100644 init_db/prompt_def.json create mode 100644 init_db/prompt_init.py diff --git a/.env-example b/.env-example index d2f32bc..c75eadd 100644 --- a/.env-example +++ b/.env-example @@ -40,5 +40,7 @@ OTEL_SERVICE_NAME="valmi-app-backend" OTEL_PYTHON_LOG_LEVEL="debug" OTEL_PYTHON_LOG_CORRELATION=True OTEL_EXPORTER_OTLP_INSECURE=True - +DB_URL="classspace.in" +DB_USERNAME="****************" +DB_PASSWORD="****************" #STREAM_API_URL="http://localhost:3100" \ No newline at end of file diff --git a/core/api.py b/core/api.py index 2c09de2..3e01b53 100644 --- a/core/api.py +++ b/core/api.py @@ -194,33 +194,6 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): logger.debug(data) logger.debug(workspace_id) source = data.get("connector_type") - # if source=="SRC_SHOPIFY": - # host_url = os.environ["DB_URL"] - # db_password = os.environ["DB_PASSWORD"] - # db_username = os.environ["DB_USERNAME"] - # conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - # cursor = conn.cursor() - # create_new_cred = True - # try: - # do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) - # create_new_cred = False - # except Exception: - # create_new_cred = True - # if create_new_cred: - # user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - # password = ''.join(random.choices(string.ascii_uppercase, k=17)) - # creds = {'username': user_name, 'password': password} - # credential_info = {"id": uuid.uuid4()} - # credential_info["workspace"] = Workspace.objects.get(id=workspace_id) - # credential_info["connector_config"] = creds - # result = StorageCredentials.objects.create(**credential_info) - # logger.debug(result) - # query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - # cursor.execute(query, (password,)) - # query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA public TO {username}").format(username=user_name) - # cursor.execute(query) - # conn.commit() - # conn.close() data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) data["connector"] = Connector.objects.get(type=data.pop("connector_type")) diff --git a/core/engine_api.py b/core/engine_api.py index 6e760ae..f9cf41e 100644 --- a/core/engine_api.py +++ b/core/engine_api.py @@ -12,10 +12,11 @@ from decouple import Csv, config from ninja import Router -from core.schemas import ConnectorSchema, DetailSchema, SyncSchema +from core.schemas import ConnectorSchema, DetailSchema, PromptSchema, SyncSchema from .models import ( Connector, + Prompt, Sync, OAuthApiKeys ) @@ -95,6 +96,18 @@ def create_connector(request, payload: ConnectorSchema): logger.exception("Connector error") return (400, {"detail": "The specific connector cannot be created."}) +@router.post("/prompts/create", response={200: PromptSchema, 400: DetailSchema}) +def create_connector(request, payload: PromptSchema): + # check for admin permissions + data = payload.dict() + logger.debug(data) + try: + logger.debug("creating connector") + prompts = Prompt.objects.create(**data) + return (200, PromptSchema) + except Exception: + logger.exception("Connector error") + return (400, {"detail": "The specific connector cannot be created."}) @router.get("/connectors/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) def get_connectors(request): diff --git a/core/migrations/0013_connector_mode.py b/core/migrations/0013_connector_mode.py deleted file mode 100644 index 6019eb2..0000000 --- a/core/migrations/0013_connector_mode.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 3.1.5 on 2024-04-02 12:07 - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0012_auto_20240212_0639'), - ] - - operations = [ - migrations.AddField( - model_name='connector', - name='mode', - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField(max_length=64), blank=True, default=list, size=None), - ), - ] diff --git a/core/migrations/0014_prompt.py b/core/migrations/0014_prompt.py new file mode 100644 index 0000000..d79de23 --- /dev/null +++ b/core/migrations/0014_prompt.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1.5 on 2024-04-11 15:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_storagecredentials'), + ] + + operations = [ + migrations.CreateModel( + name='Prompt', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/core/migrations/0015_auto_20240411_1531.py b/core/migrations/0015_auto_20240411_1531.py new file mode 100644 index 0000000..5ee0a50 --- /dev/null +++ b/core/migrations/0015_auto_20240411_1531.py @@ -0,0 +1,26 @@ +# Generated by Django 3.1.5 on 2024-04-11 15:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0014_prompt'), + ] + + operations = [ + migrations.CreateModel( + name='Query', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('display_name', models.CharField(default='DUMMY_CONNECTOR_DISPLAY_NAME', max_length=128)), + ('query', models.CharField(default='SELECT * FROM products', max_length=128)), + ], + ), + migrations.DeleteModel( + name='Prompt', + ), + ] diff --git a/core/migrations/0016_auto_20240412_0723.py b/core/migrations/0016_auto_20240412_0723.py new file mode 100644 index 0000000..4600ad5 --- /dev/null +++ b/core/migrations/0016_auto_20240412_0723.py @@ -0,0 +1,41 @@ +# Generated by Django 3.1.5 on 2024-04-12 07:23 + +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0015_auto_20240411_1531'), + ] + + operations = [ + migrations.CreateModel( + name='Package', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('id', models.UUIDField(default=uuid.UUID('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'), editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=256)), + ('scopes', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=64), blank=True, default=list, size=None)), + ], + ), + migrations.CreateModel( + name='Prompt', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('id', models.UUIDField(default=uuid.UUID('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'), editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=256)), + ('query', models.CharField(max_length=5000)), + ('parameters', models.JSONField(null=True)), + ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='prompts', to='core.package')), + ], + ), + migrations.DeleteModel( + name='Query', + ), + ] diff --git a/core/migrations/0017_auto_20240412_0730.py b/core/migrations/0017_auto_20240412_0730.py new file mode 100644 index 0000000..7639de8 --- /dev/null +++ b/core/migrations/0017_auto_20240412_0730.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-04-12 07:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0016_auto_20240412_0723'), + ] + + operations = [ + migrations.AlterField( + model_name='prompt', + name='name', + field=models.CharField(max_length=256, unique=True), + ), + ] diff --git a/core/migrations/0018_auto_20240412_1025.py b/core/migrations/0018_auto_20240412_1025.py new file mode 100644 index 0000000..429ca60 --- /dev/null +++ b/core/migrations/0018_auto_20240412_1025.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1.5 on 2024-04-12 10:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0017_auto_20240412_0730'), + ] + + operations = [ + migrations.RemoveField( + model_name='prompt', + name='created_at', + ), + migrations.RemoveField( + model_name='prompt', + name='updated_at', + ), + ] diff --git a/core/migrations/0019_auto_20240412_1228.py b/core/migrations/0019_auto_20240412_1228.py new file mode 100644 index 0000000..40ddd25 --- /dev/null +++ b/core/migrations/0019_auto_20240412_1228.py @@ -0,0 +1,40 @@ +# Generated by Django 3.1.5 on 2024-04-12 12:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0018_auto_20240412_1025'), + ] + + operations = [ + migrations.RemoveField( + model_name='package', + name='id', + ), + migrations.RemoveField( + model_name='prompt', + name='id', + ), + migrations.RemoveField( + model_name='prompt', + name='package', + ), + migrations.AddField( + model_name='prompt', + name='package_id', + field=models.CharField(default='P0', max_length=20), + ), + migrations.AlterField( + model_name='package', + name='name', + field=models.CharField(max_length=256, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='prompt', + name='name', + field=models.CharField(max_length=256, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/core/models.py b/core/models.py index d720d70..c173026 100644 --- a/core/models.py +++ b/core/models.py @@ -137,6 +137,18 @@ class Connector(models.Model): mode = ArrayField(models.CharField(max_length=64), blank=True, default=list) +class Package(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + name = models.CharField(primary_key=True,max_length=256, null=False, blank=False) + scopes = ArrayField(models.CharField(max_length=64), blank=True, default=list) + +class Prompt(models.Model): + name = models.CharField(primary_key=True,max_length=256, null=False, blank=False,unique=True) + query = models.CharField(null=False, blank = False,max_length=5000) + parameters = models.JSONField(blank=False, null=True) + package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") + class Account(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) name = models.CharField(max_length=256, null=False, blank=False) diff --git a/core/prompt_api.py b/core/prompt_api.py new file mode 100644 index 0000000..ee94c68 --- /dev/null +++ b/core/prompt_api.py @@ -0,0 +1,26 @@ +import logging + +from pydantic import Json +from core.models import Prompt +from core.schemas import DetailSchema +from ninja import Router +import json + +logger = logging.getLogger(__name__) + +router = Router() + +@router.get("/", response={200: Json, 400: DetailSchema}) +def get_prompts(request): + try: + logger.debug("listing connectors") + prompts = Prompt.objects.all() + prompts_data = list(prompts.values()) + response = json.dumps(prompts_data) + logger.info(response) + logger.info(f"prompts - {prompts}") + return response + except Exception: + logger.exception("prompts listing error") + return (400, {"detail": "The list of prompts cannot be fetched."}) + \ No newline at end of file diff --git a/core/schemas.py b/core/schemas.py index 5e25159..05c36f2 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -13,7 +13,7 @@ from ninja import Field, ModelSchema, Schema from pydantic import UUID4 -from .models import Account, Connector, Credential, Destination, Organization, Source, Sync, Workspace, OAuthApiKeys +from .models import Account, Connector, Credential, Destination, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys User = get_user_model() @@ -56,6 +56,17 @@ class Config(CamelSchemaConfig): model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] +class PackageSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Package + model_fields = ["name", "scopes"] + +class PromptSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Prompt + model_fields = ["name","query","parameters","package_id"] + + class ConnectorConfigSchemaIn(Schema): config: Dict diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index cf230db..57b074e 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -51,4 +51,5 @@ echo "from django.contrib.auth import get_user_model;\ #init the database sleep 3 python /workspace/init_db/connector_init.py +python /workspace/init_db/prompt_init.py wait diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json new file mode 100644 index 0000000..9552a15 --- /dev/null +++ b/init_db/prompt_def.json @@ -0,0 +1,13 @@ +{ + "definitions":[ + { + "name":"products", + "query":"SELECT * FROM products", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + } + ] +} \ No newline at end of file diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py new file mode 100644 index 0000000..16f4f1b --- /dev/null +++ b/init_db/prompt_init.py @@ -0,0 +1,29 @@ +""" +Copyright (c) 2024 valmi.io + +Created Date: Wednesday, April 11th 2024, 9:56:52 pm +Author: Rajashekar Varkala @ valmi.io + +""" + +from os.path import dirname, join +import json +import requests +import os +from requests.auth import HTTPBasicAuth + +prompt_defs = json.loads(open(join(dirname(__file__), "prompt_def.json"), "r").read()) + +for prompt_def in prompt_defs["definitions"]: + resp = requests.post( + f"http://localhost:{os.environ['PORT']}/api/v1/superuser/prompts/create", + json={ + "name": prompt_def["name"], + "query": prompt_def["query"], + "parameters":prompt_def["parameters"], + "package_id":prompt_def["package_id"] + }, + auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), + ) + if resp.status_code != 200: + print("Failed to create prompt. May exist already. Do better - continuing...") diff --git a/valmi_app_backend/urls.py b/valmi_app_backend/urls.py index d6afe5f..d896380 100644 --- a/valmi_app_backend/urls.py +++ b/valmi_app_backend/urls.py @@ -17,6 +17,7 @@ from core.api import router as public_api_router from core.stream_api import router as stream_api_router +from core.prompt_api import router as prompt_api_router from core.oauth_api import router as oauth_api_router @@ -120,6 +121,7 @@ def authenticate(self, request): if config("AUTHENTICATION", default=True, cast=bool): api.add_router("v1/superuser/", superuser_api_router, auth=[BasicAuth()]) api.add_router("v1/streams/", stream_api_router, auth=[AuthBearer(), BasicAuth()]) + api.add_router("v1/prompts/", prompt_api_router, auth=[AuthBearer(), BasicAuth()]) api.add_router("v1/oauth/", oauth_api_router, auth=[AuthBearer(), BasicAuth()]) api.add_router("v1/", public_api_router, auth=[AuthBearer(), BasicAuth()]) From 5e896bdb708c223ba4329674beba60f0fac0234c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 11:58:20 +0530 Subject: [PATCH 014/159] added packages --- core/engine_api.py | 24 ++++++++++-- core/explore_api.py | 45 ++++++++++++++++++++++ core/migrations/0019_auto_20240412_1228.py | 11 +----- core/migrations/0020_auto_20240415_0620.py | 31 +++++++++++++++ core/migrations/0021_auto_20240415_0727.py | 18 +++++++++ core/migrations/0022_auto_20240416_0614.py | 26 +++++++++++++ core/models.py | 15 ++++++-- core/package_api.py | 42 ++++++++++++++++++++ core/schemas.py | 14 ++++++- docker-entrypoint.sh | 1 + init_db/package_def.json | 14 +++++++ init_db/package_init.py | 31 +++++++++++++++ init_db/prompt_def.json | 11 ++++++ valmi_app_backend/urls.py | 5 ++- 14 files changed, 269 insertions(+), 19 deletions(-) create mode 100644 core/explore_api.py create mode 100644 core/migrations/0020_auto_20240415_0620.py create mode 100644 core/migrations/0021_auto_20240415_0727.py create mode 100644 core/migrations/0022_auto_20240416_0614.py create mode 100644 core/package_api.py create mode 100644 init_db/package_def.json create mode 100644 init_db/package_init.py diff --git a/core/engine_api.py b/core/engine_api.py index f9cf41e..3cc7876 100644 --- a/core/engine_api.py +++ b/core/engine_api.py @@ -12,10 +12,11 @@ from decouple import Csv, config from ninja import Router -from core.schemas import ConnectorSchema, DetailSchema, PromptSchema, SyncSchema +from core.schemas import ConnectorSchema, DetailSchema, PackageSchema, PromptSchema, SyncSchema from .models import ( Connector, + Package, Prompt, Sync, OAuthApiKeys @@ -102,12 +103,27 @@ def create_connector(request, payload: PromptSchema): data = payload.dict() logger.debug(data) try: - logger.debug("creating connector") + logger.debug("creating prompt") prompts = Prompt.objects.create(**data) return (200, PromptSchema) except Exception: - logger.exception("Connector error") - return (400, {"detail": "The specific connector cannot be created."}) + logger.exception("Prompt error") + return (400, {"detail": "The specific prompt cannot be created."}) + + +@router.post("/packages/create", response={200: PackageSchema, 400: DetailSchema}) +def create_connector(request, payload: PackageSchema): + # check for admin permissions + logger.info("logging package schema") + data = payload.dict() + logger.debug(data) + try: + logger.debug("creating package") + package = Package.objects.create(**data) + return (200, PackageSchema) + except Exception: + logger.exception("Package error") + return (400, {"detail": "The specific package cannot be created."}) @router.get("/connectors/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) def get_connectors(request): diff --git a/core/explore_api.py b/core/explore_api.py new file mode 100644 index 0000000..018ea53 --- /dev/null +++ b/core/explore_api.py @@ -0,0 +1,45 @@ +import logging +import uuid +from pydantic import Json +from core.models import Account, Explore, Prompt, Workspace +from core.schemas import DetailSchema +from ninja import Router +import json + +logger = logging.getLogger(__name__) + +router = Router() + +@router.get("/", response={200: Json, 400: DetailSchema}) +def get_prompts(request): + try: + logger.debug("listing connectors") + explores = Explore.objects.all() + explores_data_list = list(explores.values()) + response = json.dumps(explores_data_list) + return response + except Exception: + logger.exception("prompts listing error") + return (400, {"detail": "The list of explores cannot be fetched."}) + + +@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}",response={200: Json, 400: DetailSchema}) +def create_explore(request, workspace_id,prompt_id,payload: Json): + logger.info("data before creating") + data = payload.dict() + logger.info("data before creating") + logger.info(data) + try: + data["id"] = uuid.uuid4() + data["workspace"] = Workspace.objects.get(id=workspace_id) + data["prompt"] = Prompt.objects.get(id=prompt_id) + account_info = data.pop("account", None) + if account_info and len(account_info) > 0: + account_info["id"] = uuid.uuid4() + account_info["workspace"] = data["workspace"] + data["account"] = Account.objects.create(**account_info) + logger.debug(data) + explore = Explore.objects.create(**data) + except Exception: + logger.exception("explore creation error") + return (400, {"detail": "The specific explore cannot be created."}) \ No newline at end of file diff --git a/core/migrations/0019_auto_20240412_1228.py b/core/migrations/0019_auto_20240412_1228.py index 40ddd25..f2e7b23 100644 --- a/core/migrations/0019_auto_20240412_1228.py +++ b/core/migrations/0019_auto_20240412_1228.py @@ -14,10 +14,6 @@ class Migration(migrations.Migration): model_name='package', name='id', ), - migrations.RemoveField( - model_name='prompt', - name='id', - ), migrations.RemoveField( model_name='prompt', name='package', @@ -27,14 +23,9 @@ class Migration(migrations.Migration): name='package_id', field=models.CharField(default='P0', max_length=20), ), - migrations.AlterField( - model_name='package', - name='name', - field=models.CharField(max_length=256, primary_key=True, serialize=False), - ), migrations.AlterField( model_name='prompt', name='name', - field=models.CharField(max_length=256, primary_key=True, serialize=False, unique=True), + field=models.CharField(max_length=256, unique=True), ), ] diff --git a/core/migrations/0020_auto_20240415_0620.py b/core/migrations/0020_auto_20240415_0620.py new file mode 100644 index 0000000..d08f61d --- /dev/null +++ b/core/migrations/0020_auto_20240415_0620.py @@ -0,0 +1,31 @@ +# Generated by Django 3.1.5 on 2024-04-15 06:20 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0019_auto_20240412_1228'), + ] + + operations = [ + migrations.AddField( + model_name='prompt', + name='gated', + field=models.BooleanField(default=True), + ), + migrations.CreateModel( + name='Explore', + fields=[ + ('id', models.UUIDField(default=uuid.UUID('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'), editable=False, primary_key=True, serialize=False)), + ('ready', models.BooleanField(default=False)), + ('spreadsheet_url', models.URLField(blank=True, default='https://example.com', null=True)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='explore_account', to='core.account')), + ('prompt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='explore_prompt', to='core.prompt')), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='explore_workspace', to='core.workspace')), + ], + ), + ] diff --git a/core/migrations/0021_auto_20240415_0727.py b/core/migrations/0021_auto_20240415_0727.py new file mode 100644 index 0000000..dbf502f --- /dev/null +++ b/core/migrations/0021_auto_20240415_0727.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-04-15 07:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0020_auto_20240415_0620'), + ] + + operations = [ + migrations.AlterField( + model_name='package', + name='name', + field=models.CharField(max_length=256, primary_key=True,serialize=False), + ), + ] diff --git a/core/migrations/0022_auto_20240416_0614.py b/core/migrations/0022_auto_20240416_0614.py new file mode 100644 index 0000000..9445a2e --- /dev/null +++ b/core/migrations/0022_auto_20240416_0614.py @@ -0,0 +1,26 @@ +# Generated by Django 3.1.5 on 2024-04-16 06:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0021_auto_20240415_0727'), + ] + + operations = [ + migrations.RemoveField( + model_name='package', + name='created_at', + ), + migrations.RemoveField( + model_name='package', + name='updated_at', + ), + migrations.AddField( + model_name='package', + name='gated', + field=models.BooleanField(default=True), + ), + ] diff --git a/core/models.py b/core/models.py index c173026..7e97694 100644 --- a/core/models.py +++ b/core/models.py @@ -138,16 +138,17 @@ class Connector(models.Model): class Package(models.Model): - created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField(auto_now=True) name = models.CharField(primary_key=True,max_length=256, null=False, blank=False) scopes = ArrayField(models.CharField(max_length=64), blank=True, default=list) + gated = models.BooleanField(null=False, blank = False, default=True) class Prompt(models.Model): - name = models.CharField(primary_key=True,max_length=256, null=False, blank=False,unique=True) + id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) + name = models.CharField(max_length=256, null=False, blank=False,unique=True) query = models.CharField(null=False, blank = False,max_length=5000) parameters = models.JSONField(blank=False, null=True) package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") + gated = models.BooleanField(null=False, blank = False, default=True) class Account(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) @@ -158,6 +159,14 @@ class Account(models.Model): workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="accounts") +class Explore(models.Model): + id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) + workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="explore_workspace") + prompt = models.ForeignKey(to=Prompt, on_delete=models.CASCADE, related_name="explore_prompt") + ready = models.BooleanField(null=False, blank = False, default=False) + account = models.ForeignKey(to=Account, on_delete=models.CASCADE, related_name="explore_account") + spreadsheet_url = models.URLField(null=True, blank=True, default="https://example.com") + class ValmiUserIDJitsuApiToken(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True) api_token = models.CharField(max_length=256, blank=True, null=True) diff --git a/core/package_api.py b/core/package_api.py new file mode 100644 index 0000000..0637423 --- /dev/null +++ b/core/package_api.py @@ -0,0 +1,42 @@ +import logging + +from pydantic import Json +from core.models import Package +from core.schemas import DetailSchema +from ninja import Router +import json + +logger = logging.getLogger(__name__) + +router = Router() + +@router.get("/", response={200: Json, 400: DetailSchema}) +def get_packages(request): + try: + logger.debug("listing packages") + packages = Package.objects.all() + packages_data = list(packages.values()) + response = json.dumps(packages_data) + logger.info(response) + logger.info(f"packages - {packages_data}") + return response + except Exception: + logger.exception("packages listing error") + return (400, {"detail": "The list of packages cannot be fetched."}) + +@router.get("/{package_id}", response={200: Json, 400: DetailSchema}) +def get_packages(request,package_id): + try: + logger.debug("listing packages") + package_id_upper = package_id.upper() + package = Package.objects.get(name = package_id_upper) + package_dict = { + "scopes":package.scopes, + "package_id":package.name, + "gated":package.gated, + } + response = json.dumps(package_dict) + return response + except Exception: + logger.exception("package listing error") + return (400, {"detail": "Package not found"}) diff --git a/core/schemas.py b/core/schemas.py index 68cc864..0f8f25b 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -64,7 +64,7 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["name","query","parameters","package_id"] + model_fields = ["id","name","query","parameters","package_id"] class ConnectorConfigSchemaIn(Schema): @@ -85,6 +85,18 @@ class CredentialSchemaIn(Schema): account: Dict = None name: str +class ExploreSchemaIn(Schema): + ready: bool = False + spreadsheet_url:str + account: Dict = None + prompt_id:Dict = None + workspace_id:Dict = None + + +class PackageSchema(Schema): + class Config(CamelSchemaConfig): + model = Package + model_fields = ["name","scopes","gated"] class CredentialSchemaUpdateIn(Schema): id: UUID4 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 57b074e..a370981 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -52,4 +52,5 @@ echo "from django.contrib.auth import get_user_model;\ sleep 3 python /workspace/init_db/connector_init.py python /workspace/init_db/prompt_init.py +python /workspace/init_db/package_init.py wait diff --git a/init_db/package_def.json b/init_db/package_def.json new file mode 100644 index 0000000..6be5987 --- /dev/null +++ b/init_db/package_def.json @@ -0,0 +1,14 @@ +{ + "definitions":[ + { + "name":"P0", + "scopes":["orders","products"], + "gated":true + }, + { + "name":"P1", + "scopes":["orders","products"], + "gated":true + } + ] +} \ No newline at end of file diff --git a/init_db/package_init.py b/init_db/package_init.py new file mode 100644 index 0000000..9df3042 --- /dev/null +++ b/init_db/package_init.py @@ -0,0 +1,31 @@ +""" +Copyright (c) 2024 valmi.io + +Created Date: Wednesday, April 11th 2024, 9:56:52 pm +Author: Rajashekar Varkala @ valmi.io + +""" + + +from os.path import dirname, join +import json +import logging +import requests +import os +from requests.auth import HTTPBasicAuth +logger = logging.getLogger(__name__) + +package_defs = json.loads(open(join(dirname(__file__), "package_def.json"), "r").read()) + +for package_def in package_defs["definitions"]: + resp = requests.post( + f"http://localhost:{os.environ['PORT']}/api/v1/superuser/packages/create", + json={ + "name": package_def["name"], + "scopes":package_def["scopes"], + "gated":package_def["gated"], + }, + auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), + ) + if resp.status_code != 200: + print("Failed to create package. May exist already. Do better - continuing...") diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 9552a15..95e2a5e 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -1,6 +1,7 @@ { "definitions":[ { + "name":"products", "query":"SELECT * FROM products", "parameters":{ @@ -8,6 +9,16 @@ }, "package_id":"P0", "gated":true + }, + { + + "name":"Orders", + "query":"SELECT * FROM orders", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true } ] } \ No newline at end of file diff --git a/valmi_app_backend/urls.py b/valmi_app_backend/urls.py index d896380..d98f2a0 100644 --- a/valmi_app_backend/urls.py +++ b/valmi_app_backend/urls.py @@ -18,8 +18,9 @@ from core.api import router as public_api_router from core.stream_api import router as stream_api_router from core.prompt_api import router as prompt_api_router +from core.package_api import router as package_api_router from core.oauth_api import router as oauth_api_router - +from core.explore_api import router as explore_api_router from core.api import get_workspaces from core.engine_api import router as superuser_api_router @@ -122,6 +123,8 @@ def authenticate(self, request): api.add_router("v1/superuser/", superuser_api_router, auth=[BasicAuth()]) api.add_router("v1/streams/", stream_api_router, auth=[AuthBearer(), BasicAuth()]) api.add_router("v1/prompts/", prompt_api_router, auth=[AuthBearer(), BasicAuth()]) + api.add_router("v1/packages/", package_api_router, auth=[AuthBearer(), BasicAuth()]) + api.add_router("v1/explores/", explore_api_router, auth=[AuthBearer(), BasicAuth()]) api.add_router("v1/oauth/", oauth_api_router, auth=[AuthBearer(), BasicAuth()]) api.add_router("v1/", public_api_router, auth=[AuthBearer(), BasicAuth()]) From 8e11a4f3d89e4a73aef9a994c7861fbd8309be6c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 12:39:16 +0530 Subject: [PATCH 015/159] added id for prompts --- core/prompt_api.py | 7 ++++++- init_db/prompt_init.py | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index ee94c68..82522f9 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -1,4 +1,5 @@ import logging +import uuid from pydantic import Json from core.models import Prompt @@ -13,9 +14,13 @@ @router.get("/", response={200: Json, 400: DetailSchema}) def get_prompts(request): try: - logger.debug("listing connectors") + logger.debug("listing prompts") prompts = Prompt.objects.all() prompts_data = list(prompts.values()) + for prompt in prompts_data: + for key, value in prompt.items(): + if isinstance(value, uuid.UUID): + prompt[key] = str(value) response = json.dumps(prompts_data) logger.info(response) logger.info(f"prompts - {prompts}") diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index 16f4f1b..ffc6832 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -8,20 +8,24 @@ from os.path import dirname, join import json +import uuid import requests import os from requests.auth import HTTPBasicAuth + prompt_defs = json.loads(open(join(dirname(__file__), "prompt_def.json"), "r").read()) for prompt_def in prompt_defs["definitions"]: resp = requests.post( f"http://localhost:{os.environ['PORT']}/api/v1/superuser/prompts/create", json={ + "id":str(uuid.uuid4()), "name": prompt_def["name"], "query": prompt_def["query"], "parameters":prompt_def["parameters"], - "package_id":prompt_def["package_id"] + "package_id":prompt_def["package_id"], + "gated":prompt_def["gated"], }, auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), ) From 40be4214f55ca3ebe5b1130a7dee4d18ea5cc995 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 13:35:10 +0530 Subject: [PATCH 016/159] added description for prompts --- core/migrations/0023_auto_20240416_0804.py | 23 ++++++++++++++++++++++ core/models.py | 2 ++ core/schemas.py | 2 +- init_db/prompt_def.json | 2 ++ init_db/prompt_init.py | 1 + 5 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 core/migrations/0023_auto_20240416_0804.py diff --git a/core/migrations/0023_auto_20240416_0804.py b/core/migrations/0023_auto_20240416_0804.py new file mode 100644 index 0000000..f69bd78 --- /dev/null +++ b/core/migrations/0023_auto_20240416_0804.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.5 on 2024-04-16 08:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0022_auto_20240416_0614'), + ] + + operations = [ + migrations.AddField( + model_name='explore', + name='name', + field=models.CharField(default='aaaaaa', max_length=256), + ), + migrations.AddField( + model_name='prompt', + name='description', + field=models.CharField(default='aaaaaa', max_length=1000), + ), + ] diff --git a/core/models.py b/core/models.py index 7e97694..ccb0175 100644 --- a/core/models.py +++ b/core/models.py @@ -145,6 +145,7 @@ class Package(models.Model): class Prompt(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) name = models.CharField(max_length=256, null=False, blank=False,unique=True) + description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") query = models.CharField(null=False, blank = False,max_length=5000) parameters = models.JSONField(blank=False, null=True) package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") @@ -161,6 +162,7 @@ class Account(models.Model): class Explore(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) + name = models.CharField(max_length=256, null=False, blank=False,default="aaaaaa") workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="explore_workspace") prompt = models.ForeignKey(to=Prompt, on_delete=models.CASCADE, related_name="explore_prompt") ready = models.BooleanField(null=False, blank = False, default=False) diff --git a/core/schemas.py b/core/schemas.py index 0f8f25b..2d746d6 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -64,7 +64,7 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","query","parameters","package_id"] + model_fields = ["id","name","description","query","parameters","package_id"] class ConnectorConfigSchemaIn(Schema): diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 95e2a5e..d596a6b 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -3,6 +3,7 @@ { "name":"products", + "description":"Product description", "query":"SELECT * FROM products", "parameters":{ "test":"test" @@ -13,6 +14,7 @@ { "name":"Orders", + "description":"Order description", "query":"SELECT * FROM orders", "parameters":{ "test":"test" diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index ffc6832..5df841c 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -22,6 +22,7 @@ json={ "id":str(uuid.uuid4()), "name": prompt_def["name"], + "description": prompt_def["description"], "query": prompt_def["query"], "parameters":prompt_def["parameters"], "package_id":prompt_def["package_id"], From c8f6a8227c5f57dd6b68fbba6236a59df4ef8aae Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 16:06:42 +0530 Subject: [PATCH 017/159] fix for package creation --- core/engine_api.py | 4 ++-- core/schemas.py | 11 +++-------- init_db/package_init.py | 8 +++----- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/core/engine_api.py b/core/engine_api.py index 3cc7876..a18d61a 100644 --- a/core/engine_api.py +++ b/core/engine_api.py @@ -105,7 +105,7 @@ def create_connector(request, payload: PromptSchema): try: logger.debug("creating prompt") prompts = Prompt.objects.create(**data) - return (200, PromptSchema) + return (200, prompts) except Exception: logger.exception("Prompt error") return (400, {"detail": "The specific prompt cannot be created."}) @@ -120,7 +120,7 @@ def create_connector(request, payload: PackageSchema): try: logger.debug("creating package") package = Package.objects.create(**data) - return (200, PackageSchema) + return (200, package) except Exception: logger.exception("Package error") return (400, {"detail": "The specific package cannot be created."}) diff --git a/core/schemas.py b/core/schemas.py index 2d746d6..f31dc30 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -59,12 +59,12 @@ class Config(CamelSchemaConfig): class PackageSchema(ModelSchema): class Config(CamelSchemaConfig): model = Package - model_fields = ["name", "scopes"] + model_fields = ["name","gated","scopes"] class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","description","query","parameters","package_id"] + model_fields = ["id","name","description","query","parameters","package_id","gated"] class ConnectorConfigSchemaIn(Schema): @@ -87,16 +87,11 @@ class CredentialSchemaIn(Schema): class ExploreSchemaIn(Schema): ready: bool = False + name:str spreadsheet_url:str account: Dict = None - prompt_id:Dict = None - workspace_id:Dict = None -class PackageSchema(Schema): - class Config(CamelSchemaConfig): - model = Package - model_fields = ["name","scopes","gated"] class CredentialSchemaUpdateIn(Schema): id: UUID4 diff --git a/init_db/package_init.py b/init_db/package_init.py index 9df3042..86a0659 100644 --- a/init_db/package_init.py +++ b/init_db/package_init.py @@ -5,11 +5,9 @@ Author: Rajashekar Varkala @ valmi.io """ - - +import logging from os.path import dirname, join import json -import logging import requests import os from requests.auth import HTTPBasicAuth @@ -22,8 +20,8 @@ f"http://localhost:{os.environ['PORT']}/api/v1/superuser/packages/create", json={ "name": package_def["name"], - "scopes":package_def["scopes"], - "gated":package_def["gated"], + "scopes": package_def["scopes"], + "gated": package_def["gated"], }, auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), ) From 3fc2506a678231030de20a5b368b853654ec22f4 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 16:30:13 +0530 Subject: [PATCH 018/159] Replaced Json with respective schemas --- core/package_api.py | 12 ++++++------ core/prompt_api.py | 8 ++++---- core/schemas.py | 12 +++++++++++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/core/package_api.py b/core/package_api.py index 0637423..b04de2b 100644 --- a/core/package_api.py +++ b/core/package_api.py @@ -1,8 +1,8 @@ import logging +from typing import List -from pydantic import Json from core.models import Package -from core.schemas import DetailSchema +from core.schemas import DetailSchema, PackageSchema from ninja import Router import json @@ -10,7 +10,7 @@ router = Router() -@router.get("/", response={200: Json, 400: DetailSchema}) +@router.get("/", response={200: List[PackageSchema], 400: DetailSchema}) def get_packages(request): try: logger.debug("listing packages") @@ -19,12 +19,12 @@ def get_packages(request): response = json.dumps(packages_data) logger.info(response) logger.info(f"packages - {packages_data}") - return response + return packages except Exception: logger.exception("packages listing error") return (400, {"detail": "The list of packages cannot be fetched."}) -@router.get("/{package_id}", response={200: Json, 400: DetailSchema}) +@router.get("/{package_id}", response={200: PackageSchema, 400: DetailSchema}) def get_packages(request,package_id): try: logger.debug("listing packages") @@ -36,7 +36,7 @@ def get_packages(request,package_id): "gated":package.gated, } response = json.dumps(package_dict) - return response + return package except Exception: logger.exception("package listing error") return (400, {"detail": "Package not found"}) diff --git a/core/prompt_api.py b/core/prompt_api.py index 82522f9..99ce751 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -1,9 +1,9 @@ import logging +from typing import List import uuid -from pydantic import Json from core.models import Prompt -from core.schemas import DetailSchema +from core.schemas import DetailSchema, PromptSchema from ninja import Router import json @@ -11,7 +11,7 @@ router = Router() -@router.get("/", response={200: Json, 400: DetailSchema}) +@router.get("/", response={200: List[PromptSchema], 400: DetailSchema}) def get_prompts(request): try: logger.debug("listing prompts") @@ -24,7 +24,7 @@ def get_prompts(request): response = json.dumps(prompts_data) logger.info(response) logger.info(f"prompts - {prompts}") - return response + return prompts except Exception: logger.exception("prompts listing error") return (400, {"detail": "The list of prompts cannot be fetched."}) diff --git a/core/schemas.py b/core/schemas.py index f31dc30..33df393 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -13,7 +13,7 @@ from ninja import Field, ModelSchema, Schema from pydantic import UUID4 -from .models import Account, Connector, Credential, Destination, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys +from .models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys User = get_user_model() @@ -122,6 +122,16 @@ class Config(CamelSchemaConfig): account: AccountSchema = Field(None, alias="account") +class ExploreSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Explore + model_fields = ["ready", "name", "spreadsheet_url", "account", "id"] + account: AccountSchema = Field(None, alias="account") + prompt: PromptSchema = Field(None, alias="prompt") + workspace: WorkspaceSchema = Field(None, alias="workspace") + + + class BaseSchemaIn(Schema): workspace_id: UUID4 From ac3aeb85fdaba42f4e42531fc493e5b8ed71bbdf Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 16:33:26 +0530 Subject: [PATCH 019/159] Replaced Json with respective schemas --- core/package_api.py | 11 ----------- core/prompt_api.py | 10 ---------- 2 files changed, 21 deletions(-) diff --git a/core/package_api.py b/core/package_api.py index b04de2b..661ff63 100644 --- a/core/package_api.py +++ b/core/package_api.py @@ -4,7 +4,6 @@ from core.models import Package from core.schemas import DetailSchema, PackageSchema from ninja import Router -import json logger = logging.getLogger(__name__) @@ -15,10 +14,6 @@ def get_packages(request): try: logger.debug("listing packages") packages = Package.objects.all() - packages_data = list(packages.values()) - response = json.dumps(packages_data) - logger.info(response) - logger.info(f"packages - {packages_data}") return packages except Exception: logger.exception("packages listing error") @@ -30,12 +25,6 @@ def get_packages(request,package_id): logger.debug("listing packages") package_id_upper = package_id.upper() package = Package.objects.get(name = package_id_upper) - package_dict = { - "scopes":package.scopes, - "package_id":package.name, - "gated":package.gated, - } - response = json.dumps(package_dict) return package except Exception: logger.exception("package listing error") diff --git a/core/prompt_api.py b/core/prompt_api.py index 99ce751..d7cc8ef 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -1,11 +1,9 @@ import logging from typing import List -import uuid from core.models import Prompt from core.schemas import DetailSchema, PromptSchema from ninja import Router -import json logger = logging.getLogger(__name__) @@ -16,14 +14,6 @@ def get_prompts(request): try: logger.debug("listing prompts") prompts = Prompt.objects.all() - prompts_data = list(prompts.values()) - for prompt in prompts_data: - for key, value in prompt.items(): - if isinstance(value, uuid.UUID): - prompt[key] = str(value) - response = json.dumps(prompts_data) - logger.info(response) - logger.info(f"prompts - {prompts}") return prompts except Exception: logger.exception("prompts listing error") From 514fb22844af81937aa73bb68b16f1307fe1c944 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 16:48:53 +0530 Subject: [PATCH 020/159] added route for creating explores --- core/explore_api.py | 25 ++++++++++++------------- core/schemas.py | 3 ++- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 018ea53..70ddb73 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,30 +1,28 @@ import logging +from typing import List import uuid -from pydantic import Json from core.models import Account, Explore, Prompt, Workspace -from core.schemas import DetailSchema +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn from ninja import Router -import json logger = logging.getLogger(__name__) router = Router() -@router.get("/", response={200: Json, 400: DetailSchema}) -def get_prompts(request): +@router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) +def get_prompts(request,workspace_id): try: logger.debug("listing connectors") - explores = Explore.objects.all() - explores_data_list = list(explores.values()) - response = json.dumps(explores_data_list) - return response + workspace = Workspace.objects.get(id=workspace_id) + explores = Explore.objects.filter(workspace=workspace) + return explores except Exception: - logger.exception("prompts listing error") + logger.exception("explores listing error") return (400, {"detail": "The list of explores cannot be fetched."}) -@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}",response={200: Json, 400: DetailSchema}) -def create_explore(request, workspace_id,prompt_id,payload: Json): +@router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) +def create_explore(request, workspace_id,payload: ExploreSchemaIn): logger.info("data before creating") data = payload.dict() logger.info("data before creating") @@ -32,7 +30,7 @@ def create_explore(request, workspace_id,prompt_id,payload: Json): try: data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) - data["prompt"] = Prompt.objects.get(id=prompt_id) + data["prompt"] = Prompt.objects.get(id=data["prompt_id"]) account_info = data.pop("account", None) if account_info and len(account_info) > 0: account_info["id"] = uuid.uuid4() @@ -40,6 +38,7 @@ def create_explore(request, workspace_id,prompt_id,payload: Json): data["account"] = Account.objects.create(**account_info) logger.debug(data) explore = Explore.objects.create(**data) + return explore except Exception: logger.exception("explore creation error") return (400, {"detail": "The specific explore cannot be created."}) \ No newline at end of file diff --git a/core/schemas.py b/core/schemas.py index 33df393..c7e6fdc 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -90,6 +90,7 @@ class ExploreSchemaIn(Schema): name:str spreadsheet_url:str account: Dict = None + prompt_id:str @@ -129,7 +130,7 @@ class Config(CamelSchemaConfig): account: AccountSchema = Field(None, alias="account") prompt: PromptSchema = Field(None, alias="prompt") workspace: WorkspaceSchema = Field(None, alias="workspace") - + class BaseSchemaIn(Schema): From 21d2f11df84799f8cf81ded3879198b6e8934483 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 17:57:12 +0530 Subject: [PATCH 021/159] added api for getting preview data --- core/explore_api.py | 44 ++++++++++++++++++++++++++++++++++++++--- core/schemas.py | 2 ++ init_db/prompt_def.json | 8 ++++---- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 70ddb73..a9df3cf 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,8 +1,14 @@ +from datetime import datetime +import json import logging +import os from typing import List import uuid -from core.models import Account, Explore, Prompt, Workspace -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn + +import psycopg2 +from pydantic import Json +from core.models import Account, Explore, Prompt, StorageCredentials, Workspace +from core.schemas import DetailSchema, ExplorePreviewDataIn, ExploreSchema, ExploreSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -41,4 +47,36 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): return explore except Exception: logger.exception("explore creation error") - return (400, {"detail": "The specific explore cannot be created."}) \ No newline at end of file + return (400, {"detail": "The specific explore cannot be created."}) + +@router.get("/workspaces/{workspace_id}/preview-data",response={200: Json, 400: DetailSchema}) +def create_sync(request, workspace_id,payload: ExplorePreviewDataIn): + data = payload.dict() + logger.info("data before creating") + logger.info(data) + prompt = Prompt.objects.get(id=data["prompt_id"]) + storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) + host_url = os.environ["DB_URL"] + db_password = storage_cred.connector_config.get('password') + db_username = storage_cred.connector_config.get('username') + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + query = prompt.query + cursor.execute(query) + rows = cursor.fetchall() + result = [] + for row in rows: + d = {} + for i, col in enumerate(cursor.description): + for i, col in enumerate(cursor.description): + # Convert datetime objects to ISO format + if isinstance(row[i], datetime): + d[col[0]] = row[i].isoformat() + else: + d[col[0]] = row[i] + result.append(d) + + # Convert the list of dictionaries to JSON and print it + json_result = json.dumps(result) + print(json_result) + return json_result \ No newline at end of file diff --git a/core/schemas.py b/core/schemas.py index c7e6fdc..f5425dc 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -92,6 +92,8 @@ class ExploreSchemaIn(Schema): account: Dict = None prompt_id:str +class ExplorePreviewDataIn(Schema): + prompt_id:str class CredentialSchemaUpdateIn(Schema): diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index d596a6b..9916400 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -4,23 +4,23 @@ "name":"products", "description":"Product description", - "query":"SELECT * FROM products", + "query":"SELECT * FROM _airbyte_raw_products", "parameters":{ "test":"test" }, "package_id":"P0", - "gated":true + "gated":false }, { "name":"Orders", "description":"Order description", - "query":"SELECT * FROM orders", + "query":"SELECT * FROM _airbyte_raw_products", "parameters":{ "test":"test" }, "package_id":"P0", - "gated":true + "gated":false } ] } \ No newline at end of file From 65f115ee4739ab1bc2246ddc0fe7f67508ac3e7f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 16 Apr 2024 17:57:42 +0530 Subject: [PATCH 022/159] added api for getting preview data --- core/explore_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index a9df3cf..63eb75d 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -70,7 +70,7 @@ def create_sync(request, workspace_id,payload: ExplorePreviewDataIn): for i, col in enumerate(cursor.description): for i, col in enumerate(cursor.description): # Convert datetime objects to ISO format - if isinstance(row[i], datetime): + if isinstance(row[i], datetime): d[col[0]] = row[i].isoformat() else: d[col[0]] = row[i] From 6c4b69f01b2764693ed5fabf105fc496a67fde62 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 17 Apr 2024 11:42:39 +0530 Subject: [PATCH 023/159] added permissin to write for user in db --- core/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/api.py b/core/api.py index 3e01b53..23c6631 100644 --- a/core/api.py +++ b/core/api.py @@ -239,6 +239,8 @@ def get_storage_credentials(request, workspace_id): cursor.execute(query) query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) cursor.execute(query) + query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) + cursor.execute(query) conn.commit() conn.close() config = {} From e75590e55c074fb57dbe43e19977fcdf97e5a55a Mon Sep 17 00:00:00 2001 From: Nagendra Date: Wed, 17 Apr 2024 12:04:50 +0530 Subject: [PATCH 024/159] [Story]: Updated the 'prompt_def.json' file with a list of supported prompts --- init_db/prompt_def.json | 87 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 9916400..1df60e0 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -2,25 +2,100 @@ "definitions":[ { - "name":"products", - "description":"Product description", + "name":"Inventory snapshot", + "description":"Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", "query":"SELECT * FROM _airbyte_raw_products", "parameters":{ "test":"test" }, "package_id":"P0", - "gated":false + "gated":true + }, + { + + "name":"At Risk customers", + "description":"Top 100 customers who have not purchased from you in the last 30 days, ordered by their total purchase value.", + "query":"SELECT * FROM _airbyte_raw_customers", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + }, + { + + "name":"Cart abandonment", + "description":"Get a snapshot of customers who have initiated the process of making a purchase on your platform but have left before completing the transaction.", + "query":"SELECT * FROM _airbyte_raw_checkouts", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + }, + { + + "name":"Sales by Payment method", + "description":"Understand which payment methods your customers prefer. If a customer uses multiple payment methods for the same order, the order count and order value will be counted multiple times.", + "query":"SELECT * FROM _airbyte_raw_orders", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + }, + { + + "name":"Order export with products", + "description":"Receive an export of your orders with line items. Use this report to share with your fulfillment teams and identify preorder volumes.", + "query":"SELECT * FROM _airbyte_raw_orders", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true }, { + "name":"Average Order value", + "description":"Shows the average order value of all orders (excluding gift cards), divided by the total number of orders that contained at least one product other than a gift card. Order value includes taxes, shipping, and discounts before returns.", + "query":"SELECT * FROM _airbyte_raw_orders", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + }, + { "name":"Orders", - "description":"Order description", + "description":"Orders, total sales, and products sold over the specified time period. Orders include all statuses. Refunded value is removed from Total Sales on the day of the order.", + "query":"SELECT * FROM _airbyte_raw_orders", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true + }, + { + "name":"Profit by product variant over time", + "description":"Review your profits by each variant over time. Use this report to identify high margin products and loss leaders. If collaborating on products on a profit share basis, use this to help calculate payouts.", "query":"SELECT * FROM _airbyte_raw_products", "parameters":{ "test":"test" }, "package_id":"P0", - "gated":false + "gated":true + }, + { + "name":"New customer sales", + "description":"Sales metrics for new customers over the specified time-period.", + "query":"SELECT * FROM _airbyte_raw_customers", + "parameters":{ + "test":"test" + }, + "package_id":"P0", + "gated":true } + ] -} \ No newline at end of file +} From 04e14c1ff8a03ddbd4afd30cffb3eb8c7f4f6035 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 18 Apr 2024 12:07:22 +0530 Subject: [PATCH 025/159] modified preview api --- core/explore_api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 63eb75d..38b98a4 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -8,7 +8,7 @@ import psycopg2 from pydantic import Json from core.models import Account, Explore, Prompt, StorageCredentials, Workspace -from core.schemas import DetailSchema, ExplorePreviewDataIn, ExploreSchema, ExploreSchemaIn +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -44,17 +44,16 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): data["account"] = Account.objects.create(**account_info) logger.debug(data) explore = Explore.objects.create(**data) + return explore except Exception: logger.exception("explore creation error") return (400, {"detail": "The specific explore cannot be created."}) -@router.get("/workspaces/{workspace_id}/preview-data",response={200: Json, 400: DetailSchema}) -def create_sync(request, workspace_id,payload: ExplorePreviewDataIn): - data = payload.dict() +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 400: DetailSchema}) +def create_sync(request, workspace_id,prompt_id): logger.info("data before creating") - logger.info(data) - prompt = Prompt.objects.get(id=data["prompt_id"]) + prompt = Prompt.objects.get(id=prompt_id) storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) host_url = os.environ["DB_URL"] db_password = storage_cred.connector_config.get('password') From 22df6d01553926722300a3a0a503b152e0347938 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 18 Apr 2024 15:49:41 +0530 Subject: [PATCH 026/159] added end point for getting prompt by id --- core/prompt_api.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/prompt_api.py b/core/prompt_api.py index d7cc8ef..008a265 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -18,4 +18,15 @@ def get_prompts(request): except Exception: logger.exception("prompts listing error") return (400, {"detail": "The list of prompts cannot be fetched."}) + +@router.get("/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) +def get_prompts(request,prompt_id): + try: + logger.debug("listing prompts") + prompt = Prompt.objects.get(id=prompt_id) + return prompt + except Exception: + logger.exception("prompt listing error") + return (400, {"detail": "The prompt cannot be fetched."}) + \ No newline at end of file From a511ba073cfea56712314a24e78e5347ac818d47 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 22 Apr 2024 12:27:39 +0530 Subject: [PATCH 027/159] replaced schema with namespace while storing default warehouse credentials --- core/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api.py b/core/api.py index 23c6631..0659939 100644 --- a/core/api.py +++ b/core/api.py @@ -251,7 +251,7 @@ def get_storage_credentials(request, workspace_id): else: config['username'] = user_name config['password'] = password - config["schema"] = user_name + config["namespace"] = user_name config['database'] = "dvdrental" config['host'] = host_url config['port'] = 5432 From 4998d7d92bdbd43075400aa70e201262f9bf8743 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 22 Apr 2024 16:07:38 +0530 Subject: [PATCH 028/159] Moved creation of default credentials to workspace creation phase --- core/api.py | 55 ++++++--------------------------------------- core/serializers.py | 41 ++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/core/api.py b/core/api.py index 0659939..ed3f81b 100644 --- a/core/api.py +++ b/core/api.py @@ -8,21 +8,14 @@ import json import logging -import os -import random -import string import uuid -import psycopg2 import uuid - from datetime import datetime from typing import Dict, List, Optional - import requests from decouple import Csv, config from ninja import Router from pydantic import UUID4, Json - from core.schemas import ( ConnectorConfigSchemaIn, ConnectorSchema, @@ -212,48 +205,14 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): @router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) -def get_storage_credentials(request, workspace_id): - host_url = os.environ["DB_URL"] - db_password = os.environ["DB_PASSWORD"] - db_username = os.environ["DB_USERNAME"] - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - create_new_cred = True - try: - do_id_exists = StorageCredentials.objects.get(workspace_id=workspace_id) - create_new_cred = False - except Exception: - create_new_cred = True - if create_new_cred: - user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password} - credential_info = {"id": uuid.uuid4()} - credential_info["workspace"] = Workspace.objects.get(id=workspace_id) - credential_info["connector_config"] = creds - result = StorageCredentials.objects.create(**credential_info) - logger.debug(result) - query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - cursor.execute(query, (password,)) - query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) - cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) - cursor.execute(query) - query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) - cursor.execute(query) - conn.commit() - conn.close() - config = {} - if not create_new_cred: - config['username'] = do_id_exists.connector_config.get('username') - config['password'] = do_id_exists.connector_config.get('password') - config["schema"] = do_id_exists.connector_config.get("username") - else: - config['username'] = user_name - config['password'] = password - config["namespace"] = user_name +def storage_credentials(request, workspace_id): + config={} + creds = StorageCredentials.objects.get(workspace_id=workspace_id) + config['username'] = creds.connector_config["username"] + config['password'] = creds.connector_config["password"] + config["namespace"] = creds.connector_config["namespace"] config['database'] = "dvdrental" - config['host'] = host_url + config['host'] = "classspace.in" config['port'] = 5432 config["ssl"] = False return json.dumps(config) diff --git a/core/serializers.py b/core/serializers.py index 6ad6f15..5aa11e6 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -8,15 +8,18 @@ import binascii import os +import random +import string import uuid - +import logging from django.contrib.auth import authenticate, get_user_model from django.db import connection, transaction +import psycopg2 from djoser.serializers import TokenCreateSerializer, UserCreateSerializer from decouple import config from djoser.conf import settings - -from .models import Organization, Workspace, ValmiUserIDJitsuApiToken +logger = logging.getLogger(__name__) +from .models import Organization, StorageCredentials, Workspace, ValmiUserIDJitsuApiToken import hashlib User = get_user_model() @@ -66,6 +69,38 @@ def create(self, validated_data): workspace.save() user.save() user.organizations.add(org) + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] + db_username = os.environ["DATA_WAREHOUSE_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + logger.debug("logger in serializers") + + create_new_cred = True + try: + do_id_exists = StorageCredentials.objects.get(workspace_id=workspace.id) + create_new_cred = False + except Exception: + create_new_cred = True + if create_new_cred: + logger.debug("logger in creating new creds") + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password,'namespace': user_name} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace.id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + cursor.execute(query) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) + cursor.execute(query) + query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() if config("ENABLE_JITSU", default=False, cast=bool): self.patch_jitsu_user(user, workspace) From 21f8d0c25baeeb88f1e55940e7b03ac697592a20 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 23 Apr 2024 12:44:19 +0530 Subject: [PATCH 029/159] modified scopes for package P0 --- init_db/package_def.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init_db/package_def.json b/init_db/package_def.json index 6be5987..767942e 100644 --- a/init_db/package_def.json +++ b/init_db/package_def.json @@ -2,7 +2,7 @@ "definitions":[ { "name":"P0", - "scopes":["orders","products"], + "scopes":["read_all_orders","read_assigned_fulfillment_orders","read_checkouts","read_cart_transforms","read_checkout_branding_settings","read_content","read_customer_merge","read_customers","read_customer_payment_methods","read_discounts","read_draft_orders","read_gift_cards","read_fulfillments","read_gift_cards","read_inventory","read_products","read_locales","read_locations","read_orders","read_users","read_order_edits","read_customer_events","read_privacy_settings","read_validations","read_translations","read_fulfillments","read_store_credit_accounts","read_store_credit_account_transactions","read_shopify_payments_payouts","read_shopify_payments_disputes","read_shipping","read_resource_feedbacks","read_purchase_options"], "gated":true }, { From 0d3f497937ab0911848c942a1a6e3b708a3d40a5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 23 Apr 2024 12:47:15 +0530 Subject: [PATCH 030/159] Feature : added data to write into sheets(retl) flow --- core/destination_catalog.json | 34 ++++ core/explore_api.py | 175 ++++++++++++++++---- core/migrations/0024_auto_20240422_1138.py | 29 ++++ core/models.py | 4 + core/schemas.py | 2 +- core/source_catalog.json | 183 +++++++++++++++++++++ init_db/connector_def.json | 20 +++ init_db/prompt_def.json | 11 +- init_db/prompt_init.py | 1 + 9 files changed, 424 insertions(+), 35 deletions(-) create mode 100644 core/destination_catalog.json create mode 100644 core/migrations/0024_auto_20240422_1138.py create mode 100644 core/source_catalog.json diff --git a/core/destination_catalog.json b/core/destination_catalog.json new file mode 100644 index 0000000..0cf413c --- /dev/null +++ b/core/destination_catalog.json @@ -0,0 +1,34 @@ +{ + "sinks": [ + { + "name": "Google Sheets", + "label": "GoogleSheets", + "mapping": [], + "field_catalog": { + "upsert": { + "json_schema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "_airbyte_data": { + "type": "jsonb" + }, + "_airbyte_ab_id": { + "type": "character varying" + }, + "_airbyte_emitted_at": { + "type": "timestamp with time zone" + } + } + }, + "allow_freeform_fields": true, + "supported_destination_ids": [] + } + }, + "destination_sync_mode": "append", + "supported_destination_sync_modes": [ + "upsert" + ] + } + ] +} \ No newline at end of file diff --git a/core/explore_api.py b/core/explore_api.py index 38b98a4..837405f 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,13 +1,18 @@ -from datetime import datetime +import datetime import json import logging + import os +from os.path import dirname, join from typing import List import uuid - -import psycopg2 +import time +import json from pydantic import Json -from core.models import Account, Explore, Prompt, StorageCredentials, Workspace +import requests +from .models import Source, User +import psycopg2 +from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn from ninja import Router @@ -18,10 +23,9 @@ @router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) def get_prompts(request,workspace_id): try: - logger.debug("listing connectors") + logger.debug("listing explores") workspace = Workspace.objects.get(id=workspace_id) - explores = Explore.objects.filter(workspace=workspace) - return explores + return Explore.objects.filter(workspace=workspace) except Exception: logger.exception("explores listing error") return (400, {"detail": "The list of explores cannot be fetched."}) @@ -29,6 +33,8 @@ def get_prompts(request,workspace_id): @router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) def create_explore(request, workspace_id,payload: ExploreSchemaIn): + logger.info("logging rquest") + logger.info(request.user) logger.info("data before creating") data = payload.dict() logger.info("data before creating") @@ -38,44 +44,147 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): data["workspace"] = Workspace.objects.get(id=workspace_id) data["prompt"] = Prompt.objects.get(id=data["prompt_id"]) account_info = data.pop("account", None) + account = {} if account_info and len(account_info) > 0: account_info["id"] = uuid.uuid4() account_info["workspace"] = data["workspace"] - data["account"] = Account.objects.create(**account_info) + account = Account.objects.create(**account_info) + data["account"] = account logger.debug(data) - explore = Explore.objects.create(**data) - - return explore + src_credential = {"id": uuid.uuid4()} + src_credential["workspace"] = Workspace.objects.get(id=workspace_id) + src_credential["connector_id"] = "SRC_POSTGRES" + src_credential["name"] = "SRC_POSTGRES 2819" + src_credential["account"] = account + src_credential["status"] = "active" + #TODO: give nice nmaes for each creation(sync,credentials,sources,destination) + storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) + src_connector_config = { + "ssl": False, + "host": "classspace.in", + "port": 5432, + "user": storage_credential.connector_config["username"], + "database": "dvdrental", + "password": storage_credential.connector_config["password"], + "namespace": storage_credential.connector_config["namespace"], + } + src_credential["connector_config"] = src_connector_config + src_cred = Credential.objects.create(**src_credential) + logger.info(src_cred) + des_credential = {"id": uuid.uuid4()} + des_credential["workspace"] = Workspace.objects.get(id=workspace_id) + des_credential["connector_id"] = "DEST_GOOGLE-SHEETS" + des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" + des_credential["account"] = account + des_credential["status"] = "active" + des_connector_config = { + "spreadsheet_id": "https://docs.google.com/spreadsheets/d/1Smf99F4Ib_n0jUQPOPSsf-W9r3hxKckPbFzW_1JxtdE/edit?usp=sharing", + "credentials": { + "client_id": "YOUR GOOGLE CLIENT ID", + "client_secret": "YOUR GOOGLE CLIENT SECRET", + "refresh_token": "YOUR GOOGLE REFRESH TOKEN", + }, + } + des_credential["connector_config"] = des_connector_config + des_cred = Credential.objects.create(**des_credential) + source_connector = { + "name":"SRC_POSTGRES 2819", + "id":uuid.uuid4() + } + source_connector["workspace"] = Workspace.objects.get(id=workspace_id) + source_connector["credential"] = Credential.objects.get(id=src_cred.id) + source_connector_catalog = {} + json_file_path = join(dirname(__file__), 'source_catalog.json') + with open(json_file_path, 'r') as openfile: + source_connector_catalog = json.load(openfile) + source_connector["catalog"] = source_connector_catalog + source_connector["status"] = "active" + src_connector = Source.objects.create(**source_connector) + logger.info(src_connector) + time.sleep(3) + destination_connector = { + "name":"DEST_GOOGLE-SHEETS 2819", + "id":uuid.uuid4() + + } + destination_connector["workspace"] = Workspace.objects.get(id=workspace_id) + destination_connector["credential"] = Credential.objects.get(id=des_cred.id) + destination_connector_catalog = {} + json_file_path = join(dirname(__file__), 'destination_catalog.json') + with open(json_file_path, 'r') as openfile: + destination_connector_catalog = json.load(openfile) + destination_connector["catalog"] = destination_connector_catalog + des_connector = Destination.objects.create(**destination_connector) + logger.info(des_connector) + time.sleep(3) + sync_config = { + "name":"test 2819", + "id":uuid.uuid4(), + "status":"active", + "ui_state":{} + + } + schedule = {"run_interval": 3600000} + sync_config["schedule"] = schedule + sync_config["source"] = Source.objects.get(id=src_connector.id) + sync_config["destination"] = Destination.objects.get(id=des_connector.id) + sync_config["workspace"] = Workspace.objects.get(id=workspace_id) + sync = Sync.objects.create(**sync_config) + logger.info(sync) + time.sleep(10) + url = 'http://localhost:4000/api/v1/workspaces/{workspaceid}/syncs/{sync_id}/runs/create'.format(workspaceid=workspace_id, sync_id=sync.id) + logger.info(url) + user = User.objects.get(email = request.user.email) + conn = psycopg2.connect(host="host.docker.internal",port="5432",database="valmi_app",user="postgres",password="changeme") + cursor = conn.cursor() + query = f'SELECT KEY FROM authtoken_token WHERE user_id = {user.id}' + cursor.execute(query) + token_row = cursor.fetchone() + bearer_token = f'Bearer {token_row[0]}' + head = { + "Authorization":bearer_token, + 'Content-Type': 'application/json' + } + request_body = { + "full_refresh":False + } + json_body = json.dumps(request_body) + response = requests.post(url,headers=head,data=json_body) + logger.info(response) + return Explore.objects.create(**data) except Exception: logger.exception("explore creation error") return (400, {"detail": "The specific explore cannot be created."}) -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 400: DetailSchema}) -def create_sync(request, workspace_id,prompt_id): - logger.info("data before creating") + +def custom_serializer(obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview/",response={200: Json, 404: DetailSchema}) +def preview_data(request, workspace_id,prompt_id): prompt = Prompt.objects.get(id=prompt_id) storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) - host_url = os.environ["DB_URL"] + host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = storage_cred.connector_config.get('password') db_username = storage_cred.connector_config.get('username') + db_namespace = storage_cred.connector_config.get('namespace') conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) cursor = conn.cursor() - query = prompt.query - cursor.execute(query) - rows = cursor.fetchall() - result = [] - for row in rows: - d = {} - for i, col in enumerate(cursor.description): - for i, col in enumerate(cursor.description): - # Convert datetime objects to ISO format - if isinstance(row[i], datetime): - d[col[0]] = row[i].isoformat() - else: - d[col[0]] = row[i] - result.append(d) + table = prompt.table + # LOGIC for implementing pagination if necessary + # query = f'SELECT COUNT(*) FROM {db_namespace}.{table}' + # cursor.execute(query) + # count_row = cursor.fetchone() + # count = count_row[0] if count_row is not None else 0 + # page_id_int = int(page_id) - # Convert the list of dictionaries to JSON and print it - json_result = json.dumps(result) - print(json_result) - return json_result \ No newline at end of file + # if page_id_int > count/25 or page_id_int<=0: + # return (404, {"detail": "Invalid page Id."}) + # skip = 25*(page_id_int-1) + query = f'SELECT * FROM {db_namespace}.{table} LIMIT 100' + cursor.execute(query) + result = cursor.fetchall() + items = [dict(zip([key[0] for key in cursor.description],row))for row in result] + json_data = json.dumps(items, indent=4,default=custom_serializer) + return json_data \ No newline at end of file diff --git a/core/migrations/0024_auto_20240422_1138.py b/core/migrations/0024_auto_20240422_1138.py new file mode 100644 index 0000000..48b078f --- /dev/null +++ b/core/migrations/0024_auto_20240422_1138.py @@ -0,0 +1,29 @@ +# Generated by Django 3.1.5 on 2024-04-22 11:38 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0023_auto_20240416_0804'), + ] + + operations = [ + migrations.AddField( + model_name='explore', + name='created_at', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='explore', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='prompt', + name='table', + field=models.CharField(default='table_name', max_length=256), + ), + ] diff --git a/core/models.py b/core/models.py index ccb0175..af267e8 100644 --- a/core/models.py +++ b/core/models.py @@ -6,6 +6,7 @@ """ +from django.utils import timezone import uuid from django.contrib.auth import get_user_model @@ -148,6 +149,7 @@ class Prompt(models.Model): description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") query = models.CharField(null=False, blank = False,max_length=5000) parameters = models.JSONField(blank=False, null=True) + table = models.CharField(max_length=256,null=False, blank=False,default="table_name") package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) @@ -161,6 +163,8 @@ class Account(models.Model): class Explore(models.Model): + created_at = models.DateTimeField(default=timezone.now) + updated_at = models.DateTimeField(auto_now=True) id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) name = models.CharField(max_length=256, null=False, blank=False,default="aaaaaa") workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="explore_workspace") diff --git a/core/schemas.py b/core/schemas.py index f5425dc..6cabf3c 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -64,7 +64,7 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","description","query","parameters","package_id","gated"] + model_fields = ["id","name","description","query","parameters","package_id","gated","table"] class ConnectorConfigSchemaIn(Schema): diff --git a/core/source_catalog.json b/core/source_catalog.json new file mode 100644 index 0000000..0cb4e8b --- /dev/null +++ b/core/source_catalog.json @@ -0,0 +1,183 @@ +{ + "streams": [ + { + "id_key": "id", + "stream": { + "name": "dvdrental.dbt_transformer_p0.orders_with_product_data", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "image": {"type": "jsonb"}, + "product_type": {"type": "character varying"}, + "body_html": {"type": "character varying"}, + "title": {"type": "character varying"}, + "images": {"type": "jsonb"}, + "vendor": {"type": "character varying"}, + "options": {"type": "jsonb"}, + "product_id": {"type": "bigint"}, + "id": {"type": "bigint"}, + "name": {"type": "character varying"}, + "note": {"type": "character varying"}, + "tags": {"type": "character varying"}, + "test": {"type": "boolean"}, + "email": {"type": "character varying"}, + "phone": {"type": "character varying"}, + "token": {"type": "character varying"}, + "app_id": {"type": "bigint"}, + "number": {"type": "bigint"}, + "company": {"type": "character varying"}, + "refunds": {"type": "jsonb"}, + "user_id": {"type": "numeric"}, + "currency": {"type": "character varying"}, + "customer": {"type": "jsonb"}, + "shop_url": {"type": "character varying"}, + "closed_at": { + "type": "timestamp with time zone" + }, + "confirmed": {"type": "boolean"}, + "device_id": {"type": "character varying"}, + "po_number": {"type": "character varying"}, + "reference": {"type": "character varying"}, + "tax_lines": {"type": "jsonb"}, + "total_tax": {"type": "numeric"}, + "browser_ip": {"type": "character varying"}, + "cart_token": {"type": "character varying"}, + "created_at": { + "type": "timestamp with time zone" + }, + "deleted_at": { + "type": "timestamp with time zone" + }, + "line_items": {"type": "jsonb"}, + "source_url": {"type": "character varying"}, + "tax_exempt": {"type": "boolean"}, + "updated_at": { + "type": "timestamp with time zone" + }, + "checkout_id": {"type": "bigint"}, + "location_id": {"type": "bigint"}, + "source_name": {"type": "character varying"}, + "total_price": {"type": "numeric"}, + "cancelled_at": { + "type": "timestamp with time zone" + }, + "fulfillments": {"type": "jsonb"}, + "landing_site": {"type": "character varying"}, + "order_number": {"type": "bigint"}, + "processed_at": {"type": "character varying"}, + "total_weight": {"type": "bigint"}, + "cancel_reason": {"type": "character varying"}, + "contact_email": {"type": "character varying"}, + "payment_terms": {"type": "character varying"}, + "total_tax_set": {"type": "jsonb"}, + "checkout_token": { + "type": "character varying" + }, + "client_details": {"type": "jsonb"}, + "discount_codes": {"type": "jsonb"}, + "referring_site": { + "type": "character varying" + }, + "shipping_lines": {"type": "jsonb"}, + "subtotal_price": {"type": "numeric"}, + "taxes_included": {"type": "boolean"}, + "billing_address": {"type": "jsonb"}, + "customer_locale": { + "type": "character varying" + }, + "deleted_message": { + "type": "character varying" + }, + "estimated_taxes": {"type": "boolean"}, + "note_attributes": {"type": "jsonb"}, + "total_discounts": {"type": "numeric"}, + "total_price_set": {"type": "jsonb"}, + "total_price_usd": {"type": "numeric"}, + "financial_status": { + "type": "character varying" + }, + "landing_site_ref": { + "type": "character varying" + }, + "order_status_url": { + "type": "character varying" + }, + "shipping_address": {"type": "jsonb"}, + "current_total_tax": {"type": "numeric"}, + "source_identifier": { + "type": "character varying" + }, + "total_outstanding": {"type": "numeric"}, + "fulfillment_status": { + "type": "character varying" + }, + "subtotal_price_set": {"type": "jsonb"}, + "total_tip_received": {"type": "numeric"}, + "confirmation_number": { + "type": "character varying" + }, + "current_total_price": {"type": "numeric"}, + "deleted_description": { + "type": "character varying" + }, + "total_discounts_set": {"type": "jsonb"}, + "admin_graphql_api_id": { + "type": "character varying" + }, + "discount_allocations": {"type": "jsonb"}, + "presentment_currency": { + "type": "character varying" + }, + "current_total_tax_set": {"type": "jsonb"}, + "discount_applications": {"type": "jsonb"}, + "payment_gateway_names": {"type": "jsonb"}, + "current_subtotal_price": {"type": "numeric"}, + "total_line_items_price": {"type": "numeric"}, + "buyer_accepts_marketing": {"type": "boolean"}, + "current_total_discounts": {"type": "numeric"}, + "current_total_price_set": {"type": "jsonb"}, + "current_total_duties_set": { + "type": "character varying" + }, + "total_shipping_price_set": {"type": "jsonb"}, + "merchant_of_record_app_id": { + "type": "character varying" + }, + "original_total_duties_set": { + "type": "character varying" + }, + "current_subtotal_price_set": { + "type": "jsonb" + }, + "total_line_items_price_set": { + "type": "jsonb" + }, + "current_total_discounts_set": { + "type": "jsonb" + }, + "current_total_additional_fees_set": { + "type": "jsonb" + }, + "original_total_additional_fees_set": { + "type": "jsonb" + }, + "_airbyte_raw_id": { + "type": "character varying" + }, + "_airbyte_extracted_at": { + "type": "timestamp with time zone" + }, + "_airbyte_meta": {"type": "jsonb"} + } + }, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} \ No newline at end of file diff --git a/init_db/connector_def.json b/init_db/connector_def.json index 9095e33..fd08d88 100644 --- a/init_db/connector_def.json +++ b/init_db/connector_def.json @@ -29,6 +29,26 @@ "oauth": "False", "oauth_keys":"private", "mode":["etl"] + }, + { + "type": "SRC", + "unique_name": "POSTGRES", + "docker_image": "valmiio/source-postgres", + "docker_tag": "latest", + "display_name": "Postgres", + "oauth": "False", + "oauth_keys":"private", + "mode":["retl"] + }, + { + "type": "DEST", + "unique_name": "GOOGLE-SHEETS", + "docker_image": "valmiio/destination-google-sheets", + "docker_tag": "latest", + "display_name": "Google Sheets", + "oauth": "True", + "oauth_keys":"private", + "mode":["retl"] } ] } diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 1df60e0..24d70ab 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -4,7 +4,8 @@ "name":"Inventory snapshot", "description":"Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "query":"SELECT * FROM _airbyte_raw_products", + "query":"select * from dbt_transformer_p0.orders_with_product_data", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -16,6 +17,7 @@ "name":"At Risk customers", "description":"Top 100 customers who have not purchased from you in the last 30 days, ordered by their total purchase value.", "query":"SELECT * FROM _airbyte_raw_customers", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -27,6 +29,7 @@ "name":"Cart abandonment", "description":"Get a snapshot of customers who have initiated the process of making a purchase on your platform but have left before completing the transaction.", "query":"SELECT * FROM _airbyte_raw_checkouts", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -38,6 +41,7 @@ "name":"Sales by Payment method", "description":"Understand which payment methods your customers prefer. If a customer uses multiple payment methods for the same order, the order count and order value will be counted multiple times.", "query":"SELECT * FROM _airbyte_raw_orders", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -49,6 +53,7 @@ "name":"Order export with products", "description":"Receive an export of your orders with line items. Use this report to share with your fulfillment teams and identify preorder volumes.", "query":"SELECT * FROM _airbyte_raw_orders", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -60,6 +65,7 @@ "name":"Average Order value", "description":"Shows the average order value of all orders (excluding gift cards), divided by the total number of orders that contained at least one product other than a gift card. Order value includes taxes, shipping, and discounts before returns.", "query":"SELECT * FROM _airbyte_raw_orders", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -70,6 +76,7 @@ "name":"Orders", "description":"Orders, total sales, and products sold over the specified time period. Orders include all statuses. Refunded value is removed from Total Sales on the day of the order.", "query":"SELECT * FROM _airbyte_raw_orders", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -80,6 +87,7 @@ "name":"Profit by product variant over time", "description":"Review your profits by each variant over time. Use this report to identify high margin products and loss leaders. If collaborating on products on a profit share basis, use this to help calculate payouts.", "query":"SELECT * FROM _airbyte_raw_products", + "table":"orders_with_product_", "parameters":{ "test":"test" }, @@ -90,6 +98,7 @@ "name":"New customer sales", "description":"Sales metrics for new customers over the specified time-period.", "query":"SELECT * FROM _airbyte_raw_customers", + "table":"orders_with_product_", "parameters":{ "test":"test" }, diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index 5df841c..ce31d09 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -24,6 +24,7 @@ "name": prompt_def["name"], "description": prompt_def["description"], "query": prompt_def["query"], + "table":prompt_def["table"], "parameters":prompt_def["parameters"], "package_id":prompt_def["package_id"], "gated":prompt_def["gated"], From 8eab26f928fc7b5ea0051983924566444818d3bb Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 23 Apr 2024 18:58:50 +0530 Subject: [PATCH 031/159] feature added creation of spreadsheet url --- core/explore_api.py | 67 ++++++++++++++++++++++++++++++++++++++++----- core/schemas.py | 2 +- requirements.txt | 4 ++- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 837405f..f4ab63a 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,7 +1,9 @@ import datetime import json import logging - +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +import json import os from os.path import dirname, join from typing import List @@ -77,12 +79,13 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" des_credential["account"] = account des_credential["status"] = "active" + spreadsheet_url = create_spreadsheet(refresh_token=data["refresh_token"]) des_connector_config = { - "spreadsheet_id": "https://docs.google.com/spreadsheets/d/1Smf99F4Ib_n0jUQPOPSsf-W9r3hxKckPbFzW_1JxtdE/edit?usp=sharing", + "spreadsheet_id": spreadsheet_url, "credentials": { - "client_id": "YOUR GOOGLE CLIENT ID", - "client_secret": "YOUR GOOGLE CLIENT SECRET", - "refresh_token": "YOUR GOOGLE REFRESH TOKEN", + "client_id": "YOUR ID", + "client_secret": "YOUR SECRET", + "refresh_token": data["refresh_token"], }, } des_credential["connector_config"] = des_connector_config @@ -151,7 +154,11 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): json_body = json.dumps(request_body) response = requests.post(url,headers=head,data=json_body) logger.info(response) - return Explore.objects.create(**data) + del data["refresh_token"] + explore = Explore.objects.create(**data) + explore.spreadsheet_url = spreadsheet_url + explore.save() + return explore except Exception: logger.exception("explore creation error") return (400, {"detail": "The specific explore cannot be created."}) @@ -187,4 +194,50 @@ def preview_data(request, workspace_id,prompt_id): result = cursor.fetchall() items = [dict(zip([key[0] for key in cursor.description],row))for row in result] json_data = json.dumps(items, indent=4,default=custom_serializer) - return json_data \ No newline at end of file + return json_data + + + +def create_spreadsheet(refresh_token): + logger.debug("create_spreadsheet") + SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] + credentials_dict = { + "client_id": "YOUR ID", + "client_secret": "YOUR SECRET", + "refresh_token": refresh_token + } + + # Create a Credentials object from the dictionary + credentials = Credentials.from_authorized_user_info( + credentials_dict, scopes=SCOPES + ) + service = build("sheets", "v4", credentials=credentials) + + # Create the spreadsheet + spreadsheet = {"properties": {"title": "My Spreadsheet"}} + try: + spreadsheet = ( + service.spreadsheets() + .create(body=spreadsheet, fields="spreadsheetId") + .execute() + ) + spreadsheet_id = spreadsheet.get("spreadsheetId") + + # Update the sharing settings to make the spreadsheet publicly accessible + drive_service = build('drive', 'v3', credentials=credentials) + drive_service.permissions().create( + fileId=spreadsheet_id, + body={ + "role": "writer", + "type": "anyone", + "withLink": True + }, + fields="id" + ).execute() + + spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" + print(f"Spreadsheet URL: {spreadsheet_url}") + return spreadsheet_url + except Exception as e: + logger.error(f"Error creating spreadsheet: {e}") + return e diff --git a/core/schemas.py b/core/schemas.py index 6cabf3c..e01253b 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -88,9 +88,9 @@ class CredentialSchemaIn(Schema): class ExploreSchemaIn(Schema): ready: bool = False name:str - spreadsheet_url:str account: Dict = None prompt_id:str + refresh_token:str class ExplorePreviewDataIn(Schema): prompt_id:str diff --git a/requirements.txt b/requirements.txt index 1eebdb2..e0a78b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,6 @@ opentelemetry-sdk opentelemetry-instrumentation-django opentelemetry-exporter-otlp opentelemetry-instrumentation-logging -requests \ No newline at end of file +requests +google.oauth2 +googleapiclient \ No newline at end of file From 699b467dc7197cda012217e85e1041563196ca29 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 10:58:17 +0530 Subject: [PATCH 032/159] modified database connection in create explore --- core/explore_api.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index f4ab63a..bfc0db2 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,6 +1,7 @@ import datetime import json import logging +from urllib.parse import urlparse from google.oauth2.credentials import Credentials from googleapiclient.discovery import build import json @@ -83,8 +84,8 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { - "client_id": "YOUR ID", - "client_secret": "YOUR SECRET", + "client_id": os.environ["GOOGLE_CLIENT_ID"], + "client_secret": os.environ["GOOGLE_CLIENT_SECRET"], "refresh_token": data["refresh_token"], }, } @@ -138,7 +139,14 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): url = 'http://localhost:4000/api/v1/workspaces/{workspaceid}/syncs/{sync_id}/runs/create'.format(workspaceid=workspace_id, sync_id=sync.id) logger.info(url) user = User.objects.get(email = request.user.email) - conn = psycopg2.connect(host="host.docker.internal",port="5432",database="valmi_app",user="postgres",password="changeme") + result = urlparse(os.environ["DATABASE_URL"]) + # result = urlparse("postgresql://postgres:changeme@localhost/valmi_app") + username = result.username + password = result.password + database = result.path[1:] + hostname = result.hostname + port = result.port + conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) cursor = conn.cursor() query = f'SELECT KEY FROM authtoken_token WHERE user_id = {user.id}' cursor.execute(query) @@ -202,8 +210,8 @@ def create_spreadsheet(refresh_token): logger.debug("create_spreadsheet") SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] credentials_dict = { - "client_id": "YOUR ID", - "client_secret": "YOUR SECRET", + "client_id": os.environ["GOOGLE_CLIENT_ID"], + "client_secret": os.environ["GOOGLE_CLIENT_SECRET"], "refresh_token": refresh_token } @@ -231,7 +239,7 @@ def create_spreadsheet(refresh_token): "role": "writer", "type": "anyone", "withLink": True - }, + }, fields="id" ).execute() From ada304a935c4b531ebd3747b5d1a9d429ef8ce94 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 14:27:00 +0530 Subject: [PATCH 033/159] added endpoint for explore by id --- core/explore_api.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index bfc0db2..7060fe8 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -176,7 +176,7 @@ def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview/",response={200: Json, 404: DetailSchema}) +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) def preview_data(request, workspace_id,prompt_id): prompt = Prompt.objects.get(id=prompt_id) storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) @@ -205,6 +205,14 @@ def preview_data(request, workspace_id,prompt_id): return json_data +@router.get("/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) +def get_prompts(request,explore_id): + try: + logger.debug("listing explores") + return Explore.objects.get(id=explore_id) + except Exception: + logger.exception("explore listing error") + return (400, {"detail": "The explore cannot be fetched."}) def create_spreadsheet(refresh_token): logger.debug("create_spreadsheet") From f06fa8c5327b6175cab90fd47b9c4e6a1f878331 Mon Sep 17 00:00:00 2001 From: Nagendra Date: Wed, 24 Apr 2024 15:31:01 +0530 Subject: [PATCH 034/159] fix: update google oauth dependencies & invalid table name in prompts_def.json --- init_db/prompt_def.json | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 24d70ab..7c1760c 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -5,7 +5,7 @@ "name":"Inventory snapshot", "description":"Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", "query":"select * from dbt_transformer_p0.orders_with_product_data", - "table":"orders_with_product_", + "table":"orders_with_product_data", "parameters":{ "test":"test" }, diff --git a/requirements.txt b/requirements.txt index e0a78b0..453d54f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,5 @@ opentelemetry-instrumentation-django opentelemetry-exporter-otlp opentelemetry-instrumentation-logging requests -google.oauth2 -googleapiclient \ No newline at end of file +google.oauth +google-api-python-client \ No newline at end of file From 895ad6c430c5e7486687813c2b77c6032a39f4a4 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 16:06:02 +0530 Subject: [PATCH 035/159] modified spread sheet name --- core/explore_api.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 7060fe8..589b9ae 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -80,7 +80,8 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" des_credential["account"] = account des_credential["status"] = "active" - spreadsheet_url = create_spreadsheet(refresh_token=data["refresh_token"]) + name = data["prompt"].get("name") + spreadsheet_url = create_spreadsheet(name,refresh_token=data["refresh_token"]) des_connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { @@ -205,8 +206,8 @@ def preview_data(request, workspace_id,prompt_id): return json_data -@router.get("/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_prompts(request,explore_id): +@router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) +def get_prompts(request,workspace_id,explore_id): try: logger.debug("listing explores") return Explore.objects.get(id=explore_id) @@ -214,7 +215,7 @@ def get_prompts(request,explore_id): logger.exception("explore listing error") return (400, {"detail": "The explore cannot be fetched."}) -def create_spreadsheet(refresh_token): +def create_spreadsheet(name,refresh_token): logger.debug("create_spreadsheet") SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] credentials_dict = { @@ -222,15 +223,15 @@ def create_spreadsheet(refresh_token): "client_secret": os.environ["GOOGLE_CLIENT_SECRET"], "refresh_token": refresh_token } - + sheet_name = f'valmi.io {name} sheet' # Create a Credentials object from the dictionary credentials = Credentials.from_authorized_user_info( credentials_dict, scopes=SCOPES ) service = build("sheets", "v4", credentials=credentials) - + # Create the spreadsheet - spreadsheet = {"properties": {"title": "My Spreadsheet"}} + spreadsheet = {"properties": {"title": sheet_name}} try: spreadsheet = ( service.spreadsheets() From 563eccfb74f09ac4d7493bbad3ba063b6ee5972e Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 16:15:40 +0530 Subject: [PATCH 036/159] removed storing of preview data in other variable --- core/explore_api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 589b9ae..00aa07b 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -200,10 +200,8 @@ def preview_data(request, workspace_id,prompt_id): # skip = 25*(page_id_int-1) query = f'SELECT * FROM {db_namespace}.{table} LIMIT 100' cursor.execute(query) - result = cursor.fetchall() - items = [dict(zip([key[0] for key in cursor.description],row))for row in result] - json_data = json.dumps(items, indent=4,default=custom_serializer) - return json_data + items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] + return json.dumps(items, indent=4, default=custom_serializer) @router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) From f0646008b53c4eb2ba644d94147d30ab49e78842 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 17:20:18 +0530 Subject: [PATCH 037/159] bug: fix spreadsheet name creation --- core/explore_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 00aa07b..bbdad00 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -24,7 +24,7 @@ router = Router() @router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) -def get_prompts(request,workspace_id): +def get_explores(request,workspace_id): try: logger.debug("listing explores") workspace = Workspace.objects.get(id=workspace_id) @@ -45,7 +45,8 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): try: data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) - data["prompt"] = Prompt.objects.get(id=data["prompt_id"]) + prompt = Prompt.objects.get(id=data["prompt_id"]) + data["prompt"] = prompt account_info = data.pop("account", None) account = {} if account_info and len(account_info) > 0: @@ -80,7 +81,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" des_credential["account"] = account des_credential["status"] = "active" - name = data["prompt"].get("name") + name = prompt.name spreadsheet_url = create_spreadsheet(name,refresh_token=data["refresh_token"]) des_connector_config = { "spreadsheet_id": spreadsheet_url, From 88952a74432baf4d59bbf4d4e55a5a8afc875a05 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 24 Apr 2024 17:29:44 +0530 Subject: [PATCH 038/159] fate: add name for explore creation --- core/explore_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index bbdad00..5f7d0bc 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -81,7 +81,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" des_credential["account"] = account des_credential["status"] = "active" - name = prompt.name + name = "valmi.io "+prompt.name spreadsheet_url = create_spreadsheet(name,refresh_token=data["refresh_token"]) des_connector_config = { "spreadsheet_id": spreadsheet_url, @@ -165,6 +165,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): response = requests.post(url,headers=head,data=json_body) logger.info(response) del data["refresh_token"] + data["name"] = name explore = Explore.objects.create(**data) explore.spreadsheet_url = spreadsheet_url explore.save() From 55f8803a02d3fbcc5477b682b926da64db58c3fb Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Fri, 26 Apr 2024 12:19:25 +0530 Subject: [PATCH 039/159] feat: add new api for social login (#20) * feat: add new api for social login --------- Co-authored-by: Nagendra --- core/schemas.py | 17 +++++++++++ core/social_auth.py | 60 +++++++++++++++++++++++++++++++++++++++ valmi_app_backend/urls.py | 3 ++ 3 files changed, 80 insertions(+) create mode 100644 core/social_auth.py diff --git a/core/schemas.py b/core/schemas.py index e01253b..11daa21 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -239,3 +239,20 @@ class OAuthSchema(ModelSchema): class Config(CamelSchemaConfig): model = OAuthApiKeys model_fields = ["workspace", "type", "oauth_config"] + + +class SocialAccount(Schema): + provider: str + type: str + access_token: str + expires_at: int + refresh_token: str + scope: str + token_type: str + id_token: str +class SocialUser(Schema): + name: str + email: str +class SocialAuthLoginSchema(Schema): + account: SocialAccount + user: SocialUser diff --git a/core/social_auth.py b/core/social_auth.py new file mode 100644 index 0000000..bf9dbff --- /dev/null +++ b/core/social_auth.py @@ -0,0 +1,60 @@ +from ninja import Router, Schema +from rest_framework.authtoken.models import Token +from core.schemas import SocialAuthLoginSchema +from core.models import User, Organization, Workspace, OAuthApiKeys +import binascii +import os +import uuid +import logging + +router = Router() + + +logger = logging.getLogger(__name__) + + +class AuthToken(Schema): + auth_token: str + +# TODO took from serializaer. move to utils + + +def generate_key(): + return binascii.hexlify(os.urandom(20)).decode() + + +# TODO response for bad request, 400 +@router.post("/login", response={200: AuthToken}) +def login(request, payload: SocialAuthLoginSchema): + + req = payload.dict() + + user_data = req["user"] + + account = req["account"] + + email = user_data["email"] + + try: + user = User.objects.get(email=email) + except User.DoesNotExist: + user = User() + user.email = user_data["email"] + user.username = user_data["name"] + user.password = generate_key() + user.save(force_insert=True) + org = Organization(name="Default Organization", id=uuid.uuid4()) + org.save() + workspace = Workspace(name="Default Workspace", id=uuid.uuid4(), organization=org) + workspace.save() + user.save() + user.organizations.add(org) + oauth = OAuthApiKeys(workspace=workspace, type='GOOGLE_LOGIN', id=uuid.uuid4()) + oauth.oauth_config = { + "access_token": account["access_token"], + "refresh_token": account["refresh_token"], + "expires_at": account["expires_at"] + } + oauth.save() + token, _ = Token.objects.get_or_create(user=user) + return {"auth_token": token.key} diff --git a/valmi_app_backend/urls.py b/valmi_app_backend/urls.py index d98f2a0..9ad1927 100644 --- a/valmi_app_backend/urls.py +++ b/valmi_app_backend/urls.py @@ -21,6 +21,7 @@ from core.package_api import router as package_api_router from core.oauth_api import router as oauth_api_router from core.explore_api import router as explore_api_router +from core.social_auth import router as social_api_router from core.api import get_workspaces from core.engine_api import router as superuser_api_router @@ -119,6 +120,8 @@ def authenticate(self, request): urls_namespace="public_api", ) + +api.add_router("/auth/social", social_api_router) if config("AUTHENTICATION", default=True, cast=bool): api.add_router("v1/superuser/", superuser_api_router, auth=[BasicAuth()]) api.add_router("v1/streams/", stream_api_router, auth=[AuthBearer(), BasicAuth()]) From 62105ee11f0102c95f041fbe1948ba0d95930e82 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 26 Apr 2024 12:57:15 +0530 Subject: [PATCH 040/159] fix: Established Storage Credentials for Social Login --- core/explore_api.py | 8 ++++---- core/schemas.py | 2 +- core/social_auth.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 5f7d0bc..c0ae4ec 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -86,8 +86,8 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { - "client_id": os.environ["GOOGLE_CLIENT_ID"], - "client_secret": os.environ["GOOGLE_CLIENT_SECRET"], + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], "refresh_token": data["refresh_token"], }, } @@ -219,8 +219,8 @@ def create_spreadsheet(name,refresh_token): logger.debug("create_spreadsheet") SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] credentials_dict = { - "client_id": os.environ["GOOGLE_CLIENT_ID"], - "client_secret": os.environ["GOOGLE_CLIENT_SECRET"], + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], "refresh_token": refresh_token } sheet_name = f'valmi.io {name} sheet' diff --git a/core/schemas.py b/core/schemas.py index 11daa21..21f6613 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -45,7 +45,7 @@ class Config(CamelSchemaConfig): class UserSchemaOut(ModelSchema): class Config(CamelSchemaConfig): model = User - model_fields = ["first_name", "email"] + model_fields = ["first_name", "email","username"] organizations: list[OrganizationSchema] = None diff --git a/core/social_auth.py b/core/social_auth.py index bf9dbff..e7e18df 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,7 +1,11 @@ +import random +import string + +import psycopg2 from ninja import Router, Schema from rest_framework.authtoken.models import Token from core.schemas import SocialAuthLoginSchema -from core.models import User, Organization, Workspace, OAuthApiKeys +from core.models import StorageCredentials, User, Organization, Workspace, OAuthApiKeys import binascii import os import uuid @@ -48,6 +52,29 @@ def login(request, payload: SocialAuthLoginSchema): workspace = Workspace(name="Default Workspace", id=uuid.uuid4(), organization=org) workspace.save() user.save() + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] + db_username = os.environ["DATA_WAREHOUSE_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + logger.debug("logger in creating new creds") + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password,'namespace': user_name} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace.id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + cursor.execute(query) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) + cursor.execute(query) + query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() user.organizations.add(org) oauth = OAuthApiKeys(workspace=workspace, type='GOOGLE_LOGIN', id=uuid.uuid4()) oauth.oauth_config = { From 91eae7e374684675dd983df1d36009f10eb64baf Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 26 Apr 2024 13:27:42 +0530 Subject: [PATCH 041/159] fix: Established Storage Credentials for Social Login --- .env-example | 9 +++---- core/services/warehouse_credentials.py | 36 ++++++++++++++++++++++++++ core/social_auth.py | 30 +++------------------ 3 files changed, 42 insertions(+), 33 deletions(-) create mode 100644 core/services/warehouse_credentials.py diff --git a/.env-example b/.env-example index 7a78258..2bd8d9a 100644 --- a/.env-example +++ b/.env-example @@ -40,12 +40,9 @@ OTEL_SERVICE_NAME="valmi-app-backend" OTEL_PYTHON_LOG_LEVEL="debug" OTEL_PYTHON_LOG_CORRELATION=True OTEL_EXPORTER_OTLP_INSECURE=True -DB_URL="classspace.in" -DB_USERNAME="****************" -DB_PASSWORD="****************" #STREAM_API_URL="http://localhost:3100" -DB_URL="**********" -DB_USERNAME="*******" -DB_PASSWORD="*********" \ No newline at end of file +DATA_WAREHOUSE_URL="********************" +DATA_WAREHOUSE_USERNAME="****************" +DATA_WAREHOUSE_PASSWORD="***************8" \ No newline at end of file diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py new file mode 100644 index 0000000..d109599 --- /dev/null +++ b/core/services/warehouse_credentials.py @@ -0,0 +1,36 @@ + +import os +import random +import string +import uuid +import logging +import psycopg2 + +from core.models import StorageCredentials, Workspace +logger = logging.getLogger(__name__) + +class DefaultWarehouse(): + def create(workspace): + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] + db_username = os.environ["DATA_WAREHOUSE_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + logger.debug("logger in creating new creds") + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password,'namespace': user_name} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace.id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + cursor.execute(query) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) + cursor.execute(query) + query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() \ No newline at end of file diff --git a/core/social_auth.py b/core/social_auth.py index e7e18df..5a02be1 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,11 +1,9 @@ -import random -import string -import psycopg2 from ninja import Router, Schema from rest_framework.authtoken.models import Token from core.schemas import SocialAuthLoginSchema -from core.models import StorageCredentials, User, Organization, Workspace, OAuthApiKeys +from core.models import User, Organization, Workspace, OAuthApiKeys +from core.services import warehouse_credentials import binascii import os import uuid @@ -52,29 +50,7 @@ def login(request, payload: SocialAuthLoginSchema): workspace = Workspace(name="Default Workspace", id=uuid.uuid4(), organization=org) workspace.save() user.save() - host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] - db_username = os.environ["DATA_WAREHOUSE_USERNAME"] - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - logger.debug("logger in creating new creds") - user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name} - credential_info = {"id": uuid.uuid4()} - credential_info["workspace"] = Workspace.objects.get(id=workspace.id) - credential_info["connector_config"] = creds - result = StorageCredentials.objects.create(**credential_info) - query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - cursor.execute(query, (password,)) - query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) - cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) - cursor.execute(query) - query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) - cursor.execute(query) - conn.commit() - conn.close() + warehouse_credentials.DefaultWarehouse.create(workspace) user.organizations.add(org) oauth = OAuthApiKeys(workspace=workspace, type='GOOGLE_LOGIN', id=uuid.uuid4()) oauth.oauth_config = { From 4946601ca6f2f2295516b210010f403f691ef28f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 26 Apr 2024 16:08:08 +0530 Subject: [PATCH 042/159] fix: Established Storage Credentials for Social Login --- core/api.py | 1 + core/services/warehouse_credentials.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/api.py b/core/api.py index ed3f81b..89970a1 100644 --- a/core/api.py +++ b/core/api.py @@ -211,6 +211,7 @@ def storage_credentials(request, workspace_id): config['username'] = creds.connector_config["username"] config['password'] = creds.connector_config["password"] config["namespace"] = creds.connector_config["namespace"] + config["schema"] = creds.connector_config["schema"] config['database'] = "dvdrental" config['host'] = "classspace.in" config['port'] = 5432 diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py index d109599..1c60fd9 100644 --- a/core/services/warehouse_credentials.py +++ b/core/services/warehouse_credentials.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) class DefaultWarehouse(): + @staticmethod def create(workspace): host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] @@ -19,7 +20,7 @@ def create(workspace): logger.debug("logger in creating new creds") user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name} + creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name} credential_info = {"id": uuid.uuid4()} credential_info["workspace"] = Workspace.objects.get(id=workspace.id) credential_info["connector_config"] = creds From 2b6515da741862583252b243367e5e432e74a4e5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 26 Apr 2024 17:29:45 +0530 Subject: [PATCH 043/159] feature: Return organization details when creating a user --- core/schemas.py | 7 +++++++ core/social_auth.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/schemas.py b/core/schemas.py index 21f6613..6456478 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -50,6 +50,13 @@ class Config(CamelSchemaConfig): organizations: list[OrganizationSchema] = None +class CreatedUserSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = User + model_fields = ["first_name", "email","username"] + organizations: list[OrganizationSchema] = None + auth_token: str + class ConnectorSchema(ModelSchema): class Config(CamelSchemaConfig): model = Connector diff --git a/core/social_auth.py b/core/social_auth.py index 5a02be1..f56fbab 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,7 +1,7 @@ from ninja import Router, Schema from rest_framework.authtoken.models import Token -from core.schemas import SocialAuthLoginSchema +from core.schemas import CreatedUserSchema, SocialAuthLoginSchema from core.models import User, Organization, Workspace, OAuthApiKeys from core.services import warehouse_credentials import binascii @@ -26,7 +26,7 @@ def generate_key(): # TODO response for bad request, 400 -@router.post("/login", response={200: AuthToken}) +@router.post("/login", response={200: CreatedUserSchema}) def login(request, payload: SocialAuthLoginSchema): req = payload.dict() @@ -60,4 +60,14 @@ def login(request, payload: SocialAuthLoginSchema): } oauth.save() token, _ = Token.objects.get_or_create(user=user) - return {"auth_token": token.key} + user_id = user.id + queryset = User.objects.prefetch_related("organizations").get(id=user_id) + logger.debug(queryset) + response = { + "email" : user.email, + "username" : user.username, + "auth_token" :token.key, + "queryset":queryset + } + logger.debug(response) + return response From ab357b9fe4de9b908ed3ddb6d04e7ecf347b2dff Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Sun, 28 Apr 2024 15:37:58 +0530 Subject: [PATCH 044/159] fix: fix: Resolve Google login authentication issue --- core/social_auth.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/social_auth.py b/core/social_auth.py index f56fbab..33017e2 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,7 +1,7 @@ from ninja import Router, Schema from rest_framework.authtoken.models import Token -from core.schemas import CreatedUserSchema, SocialAuthLoginSchema +from core.schemas import SocialAuthLoginSchema from core.models import User, Organization, Workspace, OAuthApiKeys from core.services import warehouse_credentials import binascii @@ -26,7 +26,7 @@ def generate_key(): # TODO response for bad request, 400 -@router.post("/login", response={200: CreatedUserSchema}) +@router.post("/login", response={200: AuthToken}) def login(request, payload: SocialAuthLoginSchema): req = payload.dict() @@ -64,10 +64,7 @@ def login(request, payload: SocialAuthLoginSchema): queryset = User.objects.prefetch_related("organizations").get(id=user_id) logger.debug(queryset) response = { - "email" : user.email, - "username" : user.username, "auth_token" :token.key, - "queryset":queryset } logger.debug(response) return response From a89ac7dd3d9801ccbbd392dddf47f5a2e90ec95f Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Tue, 30 Apr 2024 16:14:14 +0530 Subject: [PATCH 045/159] Oauth login (#22) * fix: Retrieve organization credentials during OAuth login --------- Co-authored-by: supradeep2819 --- core/schemas.py | 1 + core/serializers.py | 21 ++------------------- core/social_auth.py | 31 ++++++++++++++++++++++++------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/core/schemas.py b/core/schemas.py index 6456478..488b3af 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -56,6 +56,7 @@ class Config(CamelSchemaConfig): model_fields = ["first_name", "email","username"] organizations: list[OrganizationSchema] = None auth_token: str + class ConnectorSchema(ModelSchema): class Config(CamelSchemaConfig): diff --git a/core/serializers.py b/core/serializers.py index 5aa11e6..f1cfd1c 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -8,8 +8,6 @@ import binascii import os -import random -import string import uuid import logging from django.contrib.auth import authenticate, get_user_model @@ -20,6 +18,7 @@ from djoser.conf import settings logger = logging.getLogger(__name__) from .models import Organization, StorageCredentials, Workspace, ValmiUserIDJitsuApiToken +from .services import warehouse_credentials import hashlib User = get_user_model() @@ -84,23 +83,7 @@ def create(self, validated_data): create_new_cred = True if create_new_cred: logger.debug("logger in creating new creds") - user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name} - credential_info = {"id": uuid.uuid4()} - credential_info["workspace"] = Workspace.objects.get(id=workspace.id) - credential_info["connector_config"] = creds - result = StorageCredentials.objects.create(**credential_info) - query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - cursor.execute(query, (password,)) - query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) - cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) - cursor.execute(query) - query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) - cursor.execute(query) - conn.commit() - conn.close() + warehouse_credentials.DefaultWarehouse.create(workspace) if config("ENABLE_JITSU", default=False, cast=bool): self.patch_jitsu_user(user, workspace) diff --git a/core/social_auth.py b/core/social_auth.py index 33017e2..c1f0722 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,5 +1,8 @@ +from urllib.parse import urlparse from ninja import Router, Schema +import psycopg2 +from pydantic import Json from rest_framework.authtoken.models import Token from core.schemas import SocialAuthLoginSchema from core.models import User, Organization, Workspace, OAuthApiKeys @@ -8,7 +11,7 @@ import os import uuid import logging - +import json router = Router() @@ -26,7 +29,7 @@ def generate_key(): # TODO response for bad request, 400 -@router.post("/login", response={200: AuthToken}) +@router.post("/login", response={200: Json}) def login(request, payload: SocialAuthLoginSchema): req = payload.dict() @@ -61,10 +64,24 @@ def login(request, payload: SocialAuthLoginSchema): oauth.save() token, _ = Token.objects.get_or_create(user=user) user_id = user.id - queryset = User.objects.prefetch_related("organizations").get(id=user_id) - logger.debug(queryset) - response = { - "auth_token" :token.key, + #HACK: Hardcoded everything as of now need to figure out a way to work this + result = urlparse(os.environ["DATABASE_URL"]) + username = result.username + password = result.password + database = result.path[1:] + hostname = result.hostname + port = result.port + conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) + query = f'SELECT * FROM core_user_organizations WHERE user_id = {user_id}' + cursor = conn.cursor() + cursor.execute(query) + result = cursor.fetchone() + query = f"SELECT * FROM core_workspace WHERE organization_id = '{result[2]}'" + cursor.execute(query) + result = cursor.fetchone() + response = { + "auth_token": token.key, + "workspace_id": str(result[3]) } logger.debug(response) - return response + return json.dumps(response) From f772ae4e76be29fee1573368e7612f66f4313626 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 2 May 2024 14:10:54 +0530 Subject: [PATCH 046/159] fix: removed connecting to db --- core/explore_api.py | 356 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 331 insertions(+), 25 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index c0ae4ec..ff7e32f 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -12,11 +12,13 @@ import time import json from pydantic import Json -import requests -from .models import Source, User + +from core.api import create_new_run + +from .models import OAuthApiKeys, Source import psycopg2 from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SuccessSchema, SyncStartStopSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -55,6 +57,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): account = Account.objects.create(**account_info) data["account"] = account logger.debug(data) + oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") src_credential = {"id": uuid.uuid4()} src_credential["workspace"] = Workspace.objects.get(id=workspace_id) src_credential["connector_id"] = "SRC_POSTGRES" @@ -81,14 +84,14 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" des_credential["account"] = account des_credential["status"] = "active" - name = "valmi.io "+prompt.name - spreadsheet_url = create_spreadsheet(name,refresh_token=data["refresh_token"]) + name = f"valmi.io {prompt.name}" + spreadsheet_url = create_spreadsheet(name,refresh_token=oauthkeys.oauth_config["refresh_token"]) des_connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": data["refresh_token"], + "refresh_token": oauthkeys.oauth_config["refresh_token"], }, } des_credential["connector_config"] = des_connector_config @@ -138,11 +141,199 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): sync = Sync.objects.create(**sync_config) logger.info(sync) time.sleep(10) - url = 'http://localhost:4000/api/v1/workspaces/{workspaceid}/syncs/{sync_id}/runs/create'.format(workspaceid=workspace_id, sync_id=sync.id) - logger.info(url) - user = User.objects.get(email = request.user.email) + payload = SyncStartStopSchemaIn(full_refresh=True) + response = create_new_run(request,workspace_id,sync.id,payload) + print(response) + data["name"] = name + explore = Explore.objects.create(**data) + explore.spreadsheet_url = spreadsheet_url + explore.save() + return explore + except Exception: + logger.exception("explore creation error") + return (400, {"detail": "The specific explore cannot be created."}) + + +def custom_serializer(obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) +def preview_data(request, workspace_id,prompt_id): + prompt = Prompt.objects.get(id=prompt_id) + storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = storage_cred.connector_config.get('password') + db_username = storage_cred.connector_config.get('username') + db_namespace = storage_cred.connector_config.get('namespace') + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + table = prompt.table + # LOGIC for implementing pagination if necessary + # query = f'SELECT COUNT(*) FROM {db_namespace}.{table}' + # cursor.execute(query) + # count_row = cursor.fetchone() + # count = count_row[0] if count_row is not None else 0 + # page_id_int = int(page_id) + + # if page_id_int > count/25 or page_id_int<=0: + # return (404, {"detail": "Invalid page Id."}) + # skip = 25*(page_id_int-1) + query = f'SELECT * FROM {db_namespace}.{table} LIMIT 100' + cursor.execute(query) + items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] + return json.dumps(items, indent=4, default=custom_serializer) + + +@router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) +def get_explores(request,workspace_id,explore_id): + try: + logger.debug("listing explores") + return Explore.objects.get(id=explore_id) + except Exception: + logger.exception("explore listing error") + return (400, {"detail": "The explore cannot be fetched."}) + +def create_spreadsheet(name,refresh_token): + logger.debug("create_spreadsheet") + SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] + credentials_dict = { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": refresh_token + } + sheet_name = f'{name} sheet' + # Create a Credentials object from the dictionary + credentials = Credentials.from_authorized_user_info( + credentials_dict, scopes=SCOPES + ) + service = build("sheets", "v4", credentials=credentials) + + # Create the spreadsheet + spreadsheet = {"properties": {"title": sheet_name}} + try: + spreadsheet = ( + service.spreadsheets() + .create(body=spreadsheet, fields="spreadsheetId") + .execute() + ) + spreadsheet_id = spreadsheet.get("spreadsheetId") + + #Update the sharing settings to make the spreadsheet publicly accessible + drive_service = build('drive', 'v3', credentials=credentials) + drive_service.permissions().create( + fileId=spreadsheet_id, + body={ + "role": "writer", + "type": "anyone", + "withLink": True + }, + fields="id" + ).execute() + + spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" + print(f"Spreadsheet URL: {spreadsheet_url}") + return spreadsheet_url + except Exception as e: + logger.error(f"Error creating spreadsheet: {e}") + return e + + +@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: SuccessSchema, 400: DetailSchema}) +def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): + data = payload.dict() + try: + logger.debug("getting_explore_status") + explore = Explore.objects.get(id=explore_id) + sync_id = data.get('sync_id') + result = urlparse(os.environ["DATABASE_URL"]) + username = result.username + password = result.password + database = "valmi_activation" + hostname = result.hostname + port = result.port + conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) + cursor = conn.cursor() + query = f"SELECT * FROM sync_runs WHERE sync_id = '{sync_id}' ORDER BY created_at DESC LIMIT 1" + cursor.execute(query) + result = cursor.fetchone() + columns = [column[0] for column in cursor.description] + data = dict(zip(columns, result)) + print(data.get('status')) + # if data.get('status') == 'stopped': + # CODE for re running the sync from backend + # payload = SyncStartStopSchemaIn(full_refresh=True) + # response = create_new_run(request,workspace_id,sync_id,payload) + # print(response) + # return "sync got failed. Please re-try again" + if data.get('status') == 'running': + return "sync is still running" + explore.ready = True + explore.save() + return "sync completed" + except Exception: + logger.exception("get_explore_status error") + return (400, {"detail": "The explore cannot be fetched."}) +import datetime +import json +import logging +from urllib.parse import urlparse +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +import json +import os +from os.path import dirname, join +from typing import List +import uuid +import time +import json +from pydantic import Json + +from core.api import create_new_run + +from .models import Source +import psycopg2 +from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SuccessSchema, SyncStartStopSchemaIn +from ninja import Router + +logger = logging.getLogger(__name__) + +router = Router() + +@router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) +def get_explores(request,workspace_id): + try: + logger.debug("listing explores") + workspace = Workspace.objects.get(id=workspace_id) + return Explore.objects.filter(workspace=workspace) + except Exception: + logger.exception("explores listing error") + return (400, {"detail": "The list of explores cannot be fetched."}) + + +@router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) +def create_explore(request, workspace_id,payload: ExploreSchemaIn): + logger.info("logging rquest") + logger.info(request.user) + logger.info("data before creating") + data = payload.dict() + logger.info("data before creating") + logger.info(data) + try: + data["id"] = uuid.uuid4() + data["workspace"] = Workspace.objects.get(id=workspace_id) + prompt = Prompt.objects.get(id=data["prompt_id"]) + data["prompt"] = prompt + account_info = data.pop("account", None) + account = {} + if account_info and len(account_info) > 0: + account_info["id"] = uuid.uuid4() + account_info["workspace"] = data["workspace"] + account = Account.objects.create(**account_info) + data["account"] = account + logger.debug(data) result = urlparse(os.environ["DATABASE_URL"]) - # result = urlparse("postgresql://postgres:changeme@localhost/valmi_app") username = result.username password = result.password database = result.path[1:] @@ -150,21 +341,99 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): port = result.port conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) cursor = conn.cursor() - query = f'SELECT KEY FROM authtoken_token WHERE user_id = {user.id}' + query = f"select * from core_oauth_api_keys where workspace_id = '{workspace_id}'" cursor.execute(query) - token_row = cursor.fetchone() - bearer_token = f'Bearer {token_row[0]}' - head = { - "Authorization":bearer_token, - 'Content-Type': 'application/json' + result = cursor.fetchone() + columns = [column[0] for column in cursor.description] + oauth_data = dict(zip(columns, result)) + logger.debug(oauth_data) + logger.debug(oauth_data["oauth_config"]["refresh_token"]) + src_credential = {"id": uuid.uuid4()} + src_credential["workspace"] = Workspace.objects.get(id=workspace_id) + src_credential["connector_id"] = "SRC_POSTGRES" + src_credential["name"] = "SRC_POSTGRES 2819" + src_credential["account"] = account + src_credential["status"] = "active" + #TODO: give nice nmaes for each creation(sync,credentials,sources,destination) + storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) + src_connector_config = { + "ssl": False, + "host": "classspace.in", + "port": 5432, + "user": storage_credential.connector_config["username"], + "database": "dvdrental", + "password": storage_credential.connector_config["password"], + "namespace": storage_credential.connector_config["namespace"], + } + src_credential["connector_config"] = src_connector_config + src_cred = Credential.objects.create(**src_credential) + logger.info(src_cred) + des_credential = {"id": uuid.uuid4()} + des_credential["workspace"] = Workspace.objects.get(id=workspace_id) + des_credential["connector_id"] = "DEST_GOOGLE-SHEETS" + des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" + des_credential["account"] = account + des_credential["status"] = "active" + name = f"valmi.io {prompt.name}" + spreadsheet_url = create_spreadsheet(name,refresh_token=oauth_data["oauth_config"]["refresh_token"]) + des_connector_config = { + "spreadsheet_id": spreadsheet_url, + "credentials": { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": oauth_data["oauth_config"]["refresh_token"], + }, + } + des_credential["connector_config"] = des_connector_config + des_cred = Credential.objects.create(**des_credential) + source_connector = { + "name":"SRC_POSTGRES 2819", + "id":uuid.uuid4() + } + source_connector["workspace"] = Workspace.objects.get(id=workspace_id) + source_connector["credential"] = Credential.objects.get(id=src_cred.id) + source_connector_catalog = {} + json_file_path = join(dirname(__file__), 'source_catalog.json') + with open(json_file_path, 'r') as openfile: + source_connector_catalog = json.load(openfile) + source_connector["catalog"] = source_connector_catalog + source_connector["status"] = "active" + src_connector = Source.objects.create(**source_connector) + logger.info(src_connector) + time.sleep(3) + destination_connector = { + "name":"DEST_GOOGLE-SHEETS 2819", + "id":uuid.uuid4() + } - request_body = { - "full_refresh":False + destination_connector["workspace"] = Workspace.objects.get(id=workspace_id) + destination_connector["credential"] = Credential.objects.get(id=des_cred.id) + destination_connector_catalog = {} + json_file_path = join(dirname(__file__), 'destination_catalog.json') + with open(json_file_path, 'r') as openfile: + destination_connector_catalog = json.load(openfile) + destination_connector["catalog"] = destination_connector_catalog + des_connector = Destination.objects.create(**destination_connector) + logger.info(des_connector) + time.sleep(3) + sync_config = { + "name":"test 2819", + "id":uuid.uuid4(), + "status":"active", + "ui_state":{} + } - json_body = json.dumps(request_body) - response = requests.post(url,headers=head,data=json_body) - logger.info(response) - del data["refresh_token"] + schedule = {"run_interval": 3600000} + sync_config["schedule"] = schedule + sync_config["source"] = Source.objects.get(id=src_connector.id) + sync_config["destination"] = Destination.objects.get(id=des_connector.id) + sync_config["workspace"] = Workspace.objects.get(id=workspace_id) + sync = Sync.objects.create(**sync_config) + logger.info(sync) + time.sleep(10) + payload = SyncStartStopSchemaIn(full_refresh=True) + response = create_new_run(request,workspace_id,sync.id,payload) + print(response) data["name"] = name explore = Explore.objects.create(**data) explore.spreadsheet_url = spreadsheet_url @@ -207,7 +476,7 @@ def preview_data(request, workspace_id,prompt_id): @router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_prompts(request,workspace_id,explore_id): +def get_explores(request,workspace_id,explore_id): try: logger.debug("listing explores") return Explore.objects.get(id=explore_id) @@ -223,7 +492,7 @@ def create_spreadsheet(name,refresh_token): "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], "refresh_token": refresh_token } - sheet_name = f'valmi.io {name} sheet' + sheet_name = f'{name} sheet' # Create a Credentials object from the dictionary credentials = Credentials.from_authorized_user_info( credentials_dict, scopes=SCOPES @@ -240,7 +509,7 @@ def create_spreadsheet(name,refresh_token): ) spreadsheet_id = spreadsheet.get("spreadsheetId") - # Update the sharing settings to make the spreadsheet publicly accessible + #Update the sharing settings to make the spreadsheet publicly accessible drive_service = build('drive', 'v3', credentials=credentials) drive_service.permissions().create( fileId=spreadsheet_id, @@ -258,3 +527,40 @@ def create_spreadsheet(name,refresh_token): except Exception as e: logger.error(f"Error creating spreadsheet: {e}") return e + + +@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: SuccessSchema, 400: DetailSchema}) +def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): + data = payload.dict() + try: + logger.debug("getting_explore_status") + explore = Explore.objects.get(id=explore_id) + sync_id = data.get('sync_id') + result = urlparse(os.environ["DATABASE_URL"]) + username = result.username + password = result.password + database = "valmi_activation" + hostname = result.hostname + port = result.port + conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) + cursor = conn.cursor() + query = f"SELECT * FROM sync_runs WHERE sync_id = '{sync_id}' ORDER BY created_at DESC LIMIT 1" + cursor.execute(query) + result = cursor.fetchone() + columns = [column[0] for column in cursor.description] + data = dict(zip(columns, result)) + print(data.get('status')) + # if data.get('status') == 'stopped': + # CODE for re running the sync from backend + # payload = SyncStartStopSchemaIn(full_refresh=True) + # response = create_new_run(request,workspace_id,sync_id,payload) + # print(response) + # return "sync got failed. Please re-try again" + if data.get('status') == 'running': + return "sync is still running" + explore.ready = True + explore.save() + return "sync completed" + except Exception: + logger.exception("get_explore_status error") + return (400, {"detail": "The explore cannot be fetched."}) From ca80766a6998df4d8490d47b061901255d05b4ad Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 2 May 2024 15:41:41 +0530 Subject: [PATCH 047/159] fix: --- core/explore_api.py | 9 +++++++-- core/schemas.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index ff7e32f..71d8ba2 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -9,6 +9,7 @@ from os.path import dirname, join from typing import List import uuid +from decouple import config import time import json from pydantic import Json @@ -24,6 +25,7 @@ logger = logging.getLogger(__name__) router = Router() +ACTIVATION_URL = config("ACTIVATION_SERVER") @router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) def get_explores(request,workspace_id): @@ -549,15 +551,18 @@ def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSche result = cursor.fetchone() columns = [column[0] for column in cursor.description] data = dict(zip(columns, result)) - print(data.get('status')) + status = f"{ACTIVATION_URL}/syncs/{sync_id}/status", + print(status) # if data.get('status') == 'stopped': # CODE for re running the sync from backend # payload = SyncStartStopSchemaIn(full_refresh=True) # response = create_new_run(request,workspace_id,sync_id,payload) # print(response) # return "sync got failed. Please re-try again" - if data.get('status') == 'running': + if status == 'running': return "sync is still running" + if status == 'failed': + return "sync got failed. Please re-try again" explore.ready = True explore.save() return "sync completed" diff --git a/core/schemas.py b/core/schemas.py index 6456478..0a5c5ef 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -56,6 +56,7 @@ class Config(CamelSchemaConfig): model_fields = ["first_name", "email","username"] organizations: list[OrganizationSchema] = None auth_token: str + class ConnectorSchema(ModelSchema): class Config(CamelSchemaConfig): @@ -73,7 +74,8 @@ class Config(CamelSchemaConfig): model = Prompt model_fields = ["id","name","description","query","parameters","package_id","gated","table"] - +class ExploreStatusIn(Schema): + sync_id:str class ConnectorConfigSchemaIn(Schema): config: Dict @@ -97,7 +99,6 @@ class ExploreSchemaIn(Schema): name:str account: Dict = None prompt_id:str - refresh_token:str class ExplorePreviewDataIn(Schema): prompt_id:str @@ -263,3 +264,7 @@ class SocialUser(Schema): class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser + + +class ExploreStatusSchemaIn(Schema): + sync_id: str From a70ff481dca9b17b56171a8cdf5d9cd637c4fa96 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 2 May 2024 17:09:22 +0530 Subject: [PATCH 048/159] feat: Added endpoint to retrieve explore status --- core/explore_api.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 71d8ba2..9169786 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -13,6 +13,7 @@ import time import json from pydantic import Json +import requests from core.api import create_new_run @@ -531,37 +532,25 @@ def create_spreadsheet(name,refresh_token): return e -@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: SuccessSchema, 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: str, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): data = payload.dict() try: logger.debug("getting_explore_status") explore = Explore.objects.get(id=explore_id) sync_id = data.get('sync_id') - result = urlparse(os.environ["DATABASE_URL"]) - username = result.username - password = result.password - database = "valmi_activation" - hostname = result.hostname - port = result.port - conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) - cursor = conn.cursor() - query = f"SELECT * FROM sync_runs WHERE sync_id = '{sync_id}' ORDER BY created_at DESC LIMIT 1" - cursor.execute(query) - result = cursor.fetchone() - columns = [column[0] for column in cursor.description] - data = dict(zip(columns, result)) - status = f"{ACTIVATION_URL}/syncs/{sync_id}/status", + response = requests.get(f"http://valmi-activation:8000/syncs/{sync_id}/status") + status = response.text print(status) - # if data.get('status') == 'stopped': + # if status == 'stopped': # CODE for re running the sync from backend # payload = SyncStartStopSchemaIn(full_refresh=True) # response = create_new_run(request,workspace_id,sync_id,payload) # print(response) # return "sync got failed. Please re-try again" - if status == 'running': + if status == '"running"': return "sync is still running" - if status == 'failed': + if status == '"failed"': return "sync got failed. Please re-try again" explore.ready = True explore.save() From 0f484a9268990f89fa399e6bb7478f3fc6e37c28 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 2 May 2024 17:11:09 +0530 Subject: [PATCH 049/159] feature: Added endpoint to retrieve explore status --- core/explore_api.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 9169786..aee930a 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -277,32 +277,6 @@ def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSche except Exception: logger.exception("get_explore_status error") return (400, {"detail": "The explore cannot be fetched."}) -import datetime -import json -import logging -from urllib.parse import urlparse -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build -import json -import os -from os.path import dirname, join -from typing import List -import uuid -import time -import json -from pydantic import Json - -from core.api import create_new_run - -from .models import Source -import psycopg2 -from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SuccessSchema, SyncStartStopSchemaIn -from ninja import Router - -logger = logging.getLogger(__name__) - -router = Router() @router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) def get_explores(request,workspace_id): From c00f8de9714dad9be669e529cf5b3a0995f8913d Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 2 May 2024 17:20:11 +0530 Subject: [PATCH 050/159] feature: Added endpoint to retrieve explore status --- core/explore_api.py | 267 +------------------------------------------- 1 file changed, 1 insertion(+), 266 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index aee930a..ff479f7 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,7 +1,6 @@ import datetime import json import logging -from urllib.parse import urlparse from google.oauth2.credentials import Credentials from googleapiclient.discovery import build import json @@ -20,7 +19,7 @@ from .models import OAuthApiKeys, Source import psycopg2 from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SuccessSchema, SyncStartStopSchemaIn +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SyncStartStopSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -242,270 +241,6 @@ def create_spreadsheet(name,refresh_token): return e -@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: SuccessSchema, 400: DetailSchema}) -def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): - data = payload.dict() - try: - logger.debug("getting_explore_status") - explore = Explore.objects.get(id=explore_id) - sync_id = data.get('sync_id') - result = urlparse(os.environ["DATABASE_URL"]) - username = result.username - password = result.password - database = "valmi_activation" - hostname = result.hostname - port = result.port - conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) - cursor = conn.cursor() - query = f"SELECT * FROM sync_runs WHERE sync_id = '{sync_id}' ORDER BY created_at DESC LIMIT 1" - cursor.execute(query) - result = cursor.fetchone() - columns = [column[0] for column in cursor.description] - data = dict(zip(columns, result)) - print(data.get('status')) - # if data.get('status') == 'stopped': - # CODE for re running the sync from backend - # payload = SyncStartStopSchemaIn(full_refresh=True) - # response = create_new_run(request,workspace_id,sync_id,payload) - # print(response) - # return "sync got failed. Please re-try again" - if data.get('status') == 'running': - return "sync is still running" - explore.ready = True - explore.save() - return "sync completed" - except Exception: - logger.exception("get_explore_status error") - return (400, {"detail": "The explore cannot be fetched."}) - -@router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) -def get_explores(request,workspace_id): - try: - logger.debug("listing explores") - workspace = Workspace.objects.get(id=workspace_id) - return Explore.objects.filter(workspace=workspace) - except Exception: - logger.exception("explores listing error") - return (400, {"detail": "The list of explores cannot be fetched."}) - - -@router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) -def create_explore(request, workspace_id,payload: ExploreSchemaIn): - logger.info("logging rquest") - logger.info(request.user) - logger.info("data before creating") - data = payload.dict() - logger.info("data before creating") - logger.info(data) - try: - data["id"] = uuid.uuid4() - data["workspace"] = Workspace.objects.get(id=workspace_id) - prompt = Prompt.objects.get(id=data["prompt_id"]) - data["prompt"] = prompt - account_info = data.pop("account", None) - account = {} - if account_info and len(account_info) > 0: - account_info["id"] = uuid.uuid4() - account_info["workspace"] = data["workspace"] - account = Account.objects.create(**account_info) - data["account"] = account - logger.debug(data) - result = urlparse(os.environ["DATABASE_URL"]) - username = result.username - password = result.password - database = result.path[1:] - hostname = result.hostname - port = result.port - conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) - cursor = conn.cursor() - query = f"select * from core_oauth_api_keys where workspace_id = '{workspace_id}'" - cursor.execute(query) - result = cursor.fetchone() - columns = [column[0] for column in cursor.description] - oauth_data = dict(zip(columns, result)) - logger.debug(oauth_data) - logger.debug(oauth_data["oauth_config"]["refresh_token"]) - src_credential = {"id": uuid.uuid4()} - src_credential["workspace"] = Workspace.objects.get(id=workspace_id) - src_credential["connector_id"] = "SRC_POSTGRES" - src_credential["name"] = "SRC_POSTGRES 2819" - src_credential["account"] = account - src_credential["status"] = "active" - #TODO: give nice nmaes for each creation(sync,credentials,sources,destination) - storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) - src_connector_config = { - "ssl": False, - "host": "classspace.in", - "port": 5432, - "user": storage_credential.connector_config["username"], - "database": "dvdrental", - "password": storage_credential.connector_config["password"], - "namespace": storage_credential.connector_config["namespace"], - } - src_credential["connector_config"] = src_connector_config - src_cred = Credential.objects.create(**src_credential) - logger.info(src_cred) - des_credential = {"id": uuid.uuid4()} - des_credential["workspace"] = Workspace.objects.get(id=workspace_id) - des_credential["connector_id"] = "DEST_GOOGLE-SHEETS" - des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" - des_credential["account"] = account - des_credential["status"] = "active" - name = f"valmi.io {prompt.name}" - spreadsheet_url = create_spreadsheet(name,refresh_token=oauth_data["oauth_config"]["refresh_token"]) - des_connector_config = { - "spreadsheet_id": spreadsheet_url, - "credentials": { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": oauth_data["oauth_config"]["refresh_token"], - }, - } - des_credential["connector_config"] = des_connector_config - des_cred = Credential.objects.create(**des_credential) - source_connector = { - "name":"SRC_POSTGRES 2819", - "id":uuid.uuid4() - } - source_connector["workspace"] = Workspace.objects.get(id=workspace_id) - source_connector["credential"] = Credential.objects.get(id=src_cred.id) - source_connector_catalog = {} - json_file_path = join(dirname(__file__), 'source_catalog.json') - with open(json_file_path, 'r') as openfile: - source_connector_catalog = json.load(openfile) - source_connector["catalog"] = source_connector_catalog - source_connector["status"] = "active" - src_connector = Source.objects.create(**source_connector) - logger.info(src_connector) - time.sleep(3) - destination_connector = { - "name":"DEST_GOOGLE-SHEETS 2819", - "id":uuid.uuid4() - - } - destination_connector["workspace"] = Workspace.objects.get(id=workspace_id) - destination_connector["credential"] = Credential.objects.get(id=des_cred.id) - destination_connector_catalog = {} - json_file_path = join(dirname(__file__), 'destination_catalog.json') - with open(json_file_path, 'r') as openfile: - destination_connector_catalog = json.load(openfile) - destination_connector["catalog"] = destination_connector_catalog - des_connector = Destination.objects.create(**destination_connector) - logger.info(des_connector) - time.sleep(3) - sync_config = { - "name":"test 2819", - "id":uuid.uuid4(), - "status":"active", - "ui_state":{} - - } - schedule = {"run_interval": 3600000} - sync_config["schedule"] = schedule - sync_config["source"] = Source.objects.get(id=src_connector.id) - sync_config["destination"] = Destination.objects.get(id=des_connector.id) - sync_config["workspace"] = Workspace.objects.get(id=workspace_id) - sync = Sync.objects.create(**sync_config) - logger.info(sync) - time.sleep(10) - payload = SyncStartStopSchemaIn(full_refresh=True) - response = create_new_run(request,workspace_id,sync.id,payload) - print(response) - data["name"] = name - explore = Explore.objects.create(**data) - explore.spreadsheet_url = spreadsheet_url - explore.save() - return explore - except Exception: - logger.exception("explore creation error") - return (400, {"detail": "The specific explore cannot be created."}) - - -def custom_serializer(obj): - if isinstance(obj, datetime.datetime): - return obj.isoformat() - -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) -def preview_data(request, workspace_id,prompt_id): - prompt = Prompt.objects.get(id=prompt_id) - storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) - host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = storage_cred.connector_config.get('password') - db_username = storage_cred.connector_config.get('username') - db_namespace = storage_cred.connector_config.get('namespace') - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - table = prompt.table - # LOGIC for implementing pagination if necessary - # query = f'SELECT COUNT(*) FROM {db_namespace}.{table}' - # cursor.execute(query) - # count_row = cursor.fetchone() - # count = count_row[0] if count_row is not None else 0 - # page_id_int = int(page_id) - - # if page_id_int > count/25 or page_id_int<=0: - # return (404, {"detail": "Invalid page Id."}) - # skip = 25*(page_id_int-1) - query = f'SELECT * FROM {db_namespace}.{table} LIMIT 100' - cursor.execute(query) - items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] - return json.dumps(items, indent=4, default=custom_serializer) - - -@router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_explores(request,workspace_id,explore_id): - try: - logger.debug("listing explores") - return Explore.objects.get(id=explore_id) - except Exception: - logger.exception("explore listing error") - return (400, {"detail": "The explore cannot be fetched."}) - -def create_spreadsheet(name,refresh_token): - logger.debug("create_spreadsheet") - SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] - credentials_dict = { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": refresh_token - } - sheet_name = f'{name} sheet' - # Create a Credentials object from the dictionary - credentials = Credentials.from_authorized_user_info( - credentials_dict, scopes=SCOPES - ) - service = build("sheets", "v4", credentials=credentials) - - # Create the spreadsheet - spreadsheet = {"properties": {"title": sheet_name}} - try: - spreadsheet = ( - service.spreadsheets() - .create(body=spreadsheet, fields="spreadsheetId") - .execute() - ) - spreadsheet_id = spreadsheet.get("spreadsheetId") - - #Update the sharing settings to make the spreadsheet publicly accessible - drive_service = build('drive', 'v3', credentials=credentials) - drive_service.permissions().create( - fileId=spreadsheet_id, - body={ - "role": "writer", - "type": "anyone", - "withLink": True - }, - fields="id" - ).execute() - - spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" - print(f"Spreadsheet URL: {spreadsheet_url}") - return spreadsheet_url - except Exception as e: - logger.error(f"Error creating spreadsheet: {e}") - return e - - @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: str, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): data = payload.dict() From 5307d01755b5f6a6051d6c397bea0d5dfad2597b Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 3 May 2024 10:52:57 +0530 Subject: [PATCH 051/159] feat: Add foreign key constraint for sync_id while creating explores --- core/explore_api.py | 12 +++++++----- core/migrations/0025_explore_sync.py | 20 ++++++++++++++++++++ core/migrations/0026_auto_20240503_0513.py | 19 +++++++++++++++++++ core/models.py | 1 + 4 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 core/migrations/0025_explore_sync.py create mode 100644 core/migrations/0026_auto_20240503_0513.py diff --git a/core/explore_api.py b/core/explore_api.py index ff479f7..d900a8b 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -19,7 +19,7 @@ from .models import OAuthApiKeys, Source import psycopg2 from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, ExploreStatusSchemaIn, SyncStartStopSchemaIn +from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -147,6 +147,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): response = create_new_run(request,workspace_id,sync.id,payload) print(response) data["name"] = name + data["sync"] = sync explore = Explore.objects.create(**data) explore.spreadsheet_url = spreadsheet_url explore.save() @@ -242,13 +243,14 @@ def create_spreadsheet(name,refresh_token): @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: str, 400: DetailSchema}) -def get_explore_status(request,workspace_id,explore_id,payload:ExploreStatusSchemaIn): - data = payload.dict() +def get_explore_status(request,workspace_id,explore_id): try: logger.debug("getting_explore_status") explore = Explore.objects.get(id=explore_id) - sync_id = data.get('sync_id') - response = requests.get(f"http://valmi-activation:8000/syncs/{sync_id}/status") + if explore.ready: + return "sync completed" + sync_id = explore.sync.id + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last/run/status") status = response.text print(status) # if status == 'stopped': diff --git a/core/migrations/0025_explore_sync.py b/core/migrations/0025_explore_sync.py new file mode 100644 index 0000000..1405619 --- /dev/null +++ b/core/migrations/0025_explore_sync.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.5 on 2024-05-03 05:12 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0024_auto_20240422_1138'), + ] + + operations = [ + migrations.AddField( + model_name='explore', + name='sync', + field=models.ForeignKey(default=uuid.UUID('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'), on_delete=django.db.models.deletion.CASCADE, related_name='explore_sync', to='core.sync'), + ), + ] diff --git a/core/migrations/0026_auto_20240503_0513.py b/core/migrations/0026_auto_20240503_0513.py new file mode 100644 index 0000000..2c69b46 --- /dev/null +++ b/core/migrations/0026_auto_20240503_0513.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.5 on 2024-05-03 05:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_explore_sync'), + ] + + operations = [ + migrations.AlterField( + model_name='explore', + name='sync', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='explore_sync', to='core.sync'), + ), + ] diff --git a/core/models.py b/core/models.py index af267e8..19ef82a 100644 --- a/core/models.py +++ b/core/models.py @@ -169,6 +169,7 @@ class Explore(models.Model): name = models.CharField(max_length=256, null=False, blank=False,default="aaaaaa") workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="explore_workspace") prompt = models.ForeignKey(to=Prompt, on_delete=models.CASCADE, related_name="explore_prompt") + sync = models.ForeignKey(to=Sync, on_delete=models.CASCADE, related_name="explore_sync") ready = models.BooleanField(null=False, blank = False, default=False) account = models.ForeignKey(to=Account, on_delete=models.CASCADE, related_name="explore_account") spreadsheet_url = models.URLField(null=True, blank=True, default="https://example.com") From d74c8125439467297c1e560027e5c141927f6fd2 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 3 May 2024 11:09:16 +0530 Subject: [PATCH 052/159] fix: Retrieve organization credentials during OAuth login --- core/social_auth.py | 93 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/core/social_auth.py b/core/social_auth.py index c1f0722..3040628 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -4,7 +4,7 @@ import psycopg2 from pydantic import Json from rest_framework.authtoken.models import Token -from core.schemas import SocialAuthLoginSchema +from core.schemas import DetailSchema, SocialAuthLoginSchema from core.models import User, Organization, Workspace, OAuthApiKeys from core.services import warehouse_credentials import binascii @@ -29,7 +29,7 @@ def generate_key(): # TODO response for bad request, 400 -@router.post("/login", response={200: Json}) +@router.post("/login", response={200: Json,400:DetailSchema}) def login(request, payload: SocialAuthLoginSchema): req = payload.dict() @@ -64,24 +64,71 @@ def login(request, payload: SocialAuthLoginSchema): oauth.save() token, _ = Token.objects.get_or_create(user=user) user_id = user.id - #HACK: Hardcoded everything as of now need to figure out a way to work this - result = urlparse(os.environ["DATABASE_URL"]) - username = result.username - password = result.password - database = result.path[1:] - hostname = result.hostname - port = result.port - conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) - query = f'SELECT * FROM core_user_organizations WHERE user_id = {user_id}' - cursor = conn.cursor() - cursor.execute(query) - result = cursor.fetchone() - query = f"SELECT * FROM core_workspace WHERE organization_id = '{result[2]}'" - cursor.execute(query) - result = cursor.fetchone() - response = { - "auth_token": token.key, - "workspace_id": str(result[3]) - } - logger.debug(response) - return json.dumps(response) + try: + result = urlparse(os.environ["DATABASE_URL"]) + username = result.username + password = result.password + database = result.path[1:] + hostname = result.hostname + port = result.port + conn = psycopg2.connect(user=username, password=password, host=hostname, port=port,database=database) + cursor = conn.cursor() + query = """ + SELECT + json_build_object( + 'organizations', json_agg( + json_build_object( + 'created_at', organization_created_at, + 'updated_at', organization_updated_at, + 'name', organization_name, + 'id', organization_id, + 'status', organization_status, + 'workspaces', workspaces + ) + ) + ) AS json_output + FROM ( + SELECT + "core_organization"."created_at" AS "organization_created_at", + "core_organization"."updated_at" AS "organization_updated_at", + "core_organization"."name" AS "organization_name", + "core_organization"."id" AS "organization_id", + "core_organization"."status" AS "organization_status", + json_agg( + json_build_object( + 'created_at', "core_workspace"."created_at", + 'updated_at', "core_workspace"."updated_at", + 'name', "core_workspace"."name", + 'id', "core_workspace"."id", + 'organization', "core_workspace"."organization_id", + 'status', "core_workspace"."status" + ) + ) AS workspaces + FROM + "core_organization" + LEFT JOIN + "core_user_organizations" ON "core_organization"."id" = "core_user_organizations"."organization_id" + LEFT JOIN + "core_workspace" ON "core_organization"."id" = "core_workspace"."organization_id" + WHERE + "core_user_organizations"."user_id" = %s + GROUP BY + "core_organization"."created_at", + "core_organization"."updated_at", + "core_organization"."name", + "core_organization"."id", + "core_organization"."status" + ) AS subquery; + """ + cursor.execute(query,(user_id,)) + result = cursor.fetchone() + cursor.close() + conn.close() + response = { + "auth_token": token.key, + "organizations":result + } + logger.debug(response) + return json.dumps(response) + except Exception as e: + return (400, {"detail": e.message}) \ No newline at end of file From 58c6acec89ed116a4c0eb152a79be2783583fcf8 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 3 May 2024 13:23:05 +0530 Subject: [PATCH 053/159] fix: retriving sync's last run deatils --- core/explore_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index d900a8b..7762c18 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -250,7 +250,7 @@ def get_explore_status(request,workspace_id,explore_id): if explore.ready: return "sync completed" sync_id = explore.sync.id - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last/run/status") + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/lastRunStatus") status = response.text print(status) # if status == 'stopped': From 721387ecfea265d42502e69f485eb8d2dbbc22da Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 3 May 2024 13:57:38 +0530 Subject: [PATCH 054/159] fix: retriving last run ID in descending order --- core/explore_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index 7762c18..6bb805f 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -250,7 +250,7 @@ def get_explore_status(request,workspace_id,explore_id): if explore.ready: return "sync completed" sync_id = explore.sync.id - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/lastRunStatus") + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latestRunStatus") status = response.text print(status) # if status == 'stopped': From 2f6651101d2e5fe4119d71094000b68b2ce0c019 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 3 May 2024 16:20:38 +0530 Subject: [PATCH 055/159] refactor: Replaced latest sync run state as run manager states --- core/explore_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 6bb805f..cd72782 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -242,7 +242,7 @@ def create_spreadsheet(name,refresh_token): return e -@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: str, 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id): try: logger.debug("getting_explore_status") @@ -259,13 +259,17 @@ def get_explore_status(request,workspace_id,explore_id): # response = create_new_run(request,workspace_id,sync_id,payload) # print(response) # return "sync got failed. Please re-try again" + response = {} if status == '"running"': - return "sync is still running" + response["status"] = "running" + return json.dumps(response) if status == '"failed"': - return "sync got failed. Please re-try again" + response["status"] = "failed" + return json.dumps(response) explore.ready = True explore.save() - return "sync completed" + response["status"] = "success" + return json.dumps(response) except Exception: logger.exception("get_explore_status error") return (400, {"detail": "The explore cannot be fetched."}) From 0869714035bb71b60e7f1c061726fbcee8f50549 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 6 May 2024 13:32:29 +0530 Subject: [PATCH 056/159] feature: User can add multiple shopify stores --- core/api.py | 55 ++++++++++++++++++++------ core/schemas.py | 12 +++++- core/services/warehouse_credentials.py | 4 +- core/social_auth.py | 1 - init_db/connector_def.json | 2 +- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/core/api.py b/core/api.py index 89970a1..dff1155 100644 --- a/core/api.py +++ b/core/api.py @@ -17,6 +17,7 @@ from ninja import Router from pydantic import UUID4, Json from core.schemas import ( + ConnectionSchemaIn, ConnectorConfigSchemaIn, ConnectorSchema, CredentialSchema, @@ -37,11 +38,9 @@ UserSchemaOut, CreateConfigSchemaIn ) - from .models import Account, Connector, Credential, Destination, Source, StorageCredentials, Sync, User, Workspace, OAuthApiKeys - from valmi_app_backend.utils import replace_values_in_json - +from core.services import warehouse_credentials router = Router() @@ -204,18 +203,50 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): return {"detail": "The specific credential cannot be created."} +@router.post("/workspaces/{workspace_id}/connection/DefaultWarehouse", response={200: SuccessSchema, 400: DetailSchema}) +def create_connection_with_default_warehouse(request, workspace_id,payload: ConnectionSchemaIn): + data = payload.dict() + try: + source_credential_payload = CredentialSchemaIn(name=data["shopify_store"],account=data["account"],connector_type=data["source_connector_type"],connector_config=data["source_connector_config"]) + source_credential = create_credential(request,workspace_id,source_credential_payload) + workspace = Workspace.objects.get(id = workspace_id) + warehouse_credentials.DefaultWarehouse.create(workspace,data["shopify_store"]) + storage_credentials = StorageCredentials.objects.filter(workspace_id=workspace_id).get( + connector_config__shopify_store=data["shopify_store"] + ) + destination_credential_payload = CredentialSchemaIn(name="default warehouse",account=data["account"],connector_type="DEST_POSTGRES-DEST",connector_config=storage_credentials.connector_config) + destination_credential = create_credential(request,workspace_id,destination_credential_payload) + source_payload = SourceSchemaIn(name="shopify",credential_id=source_credential.id,catalog = data["source_catalog"]) + source = create_source(request,workspace_id,source_payload) + destination_payload = DestinationSchemaIn(name="default warehouse",credential_id=destination_credential.id,catalog = data["destination_catalog"]) + destination = create_destination(request,workspace_id,destination_payload) + sync_payload = SyncSchemaIn(name="shopify to default warehouse",source_id=source.id,destination_id=destination.id,schedule=data["schedule"]) + sync = create_sync(request,workspace_id,sync_payload) + run_payload = SyncStartStopSchemaIn(full_refresh=True) + response = create_new_run(request,workspace_id,sync.id,run_payload) + logger.debug(response) + return "starting sync from shopify to default warehouse" + except Exception as e: + logger.debug(e.message) + return {"detail": "The specific connection cannot be created."} + @router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) def storage_credentials(request, workspace_id): config={} - creds = StorageCredentials.objects.get(workspace_id=workspace_id) - config['username'] = creds.connector_config["username"] - config['password'] = creds.connector_config["password"] - config["namespace"] = creds.connector_config["namespace"] - config["schema"] = creds.connector_config["schema"] - config['database'] = "dvdrental" - config['host'] = "classspace.in" - config['port'] = 5432 - config["ssl"] = False + try: + creds = StorageCredentials.objects.filter(workspace_id=workspace_id).get( + connector_config__shopify_store="chitumalla-store" + ) + config['username'] = creds.connector_config["username"] + config['password'] = creds.connector_config["password"] + config["namespace"] = creds.connector_config["namespace"] + config["schema"] = creds.connector_config["schema"] + config['database'] = "dvdrental" + config['host'] = "classspace.in" + config['port'] = 5432 + config["ssl"] = False + except StorageCredentials.DoesNotExist: + return {"detail": "The specific credential cannot be found."} return json.dumps(config) diff --git a/core/schemas.py b/core/schemas.py index 0a5c5ef..387e43e 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -12,7 +12,6 @@ from django.contrib.auth import get_user_model from ninja import Field, ModelSchema, Schema from pydantic import UUID4 - from .models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys @@ -94,6 +93,17 @@ class CredentialSchemaIn(Schema): account: Dict = None name: str +class ConnectionSchemaIn(Schema): + account: Dict = None + shopify_store: str + source_catalog: Dict + destination_catalog: Dict + schedule: Dict + source_connector_type: str + source_connector_config: Dict + + + class ExploreSchemaIn(Schema): ready: bool = False name:str diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py index 1c60fd9..40619d8 100644 --- a/core/services/warehouse_credentials.py +++ b/core/services/warehouse_credentials.py @@ -11,7 +11,7 @@ class DefaultWarehouse(): @staticmethod - def create(workspace): + def create(workspace,shopify_store): host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] db_username = os.environ["DATA_WAREHOUSE_USERNAME"] @@ -20,7 +20,7 @@ def create(workspace): logger.debug("logger in creating new creds") user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name} + creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name,'shopify_store':shopify_store,'host':'classspace.in','database':'dvdrental','port':5432} credential_info = {"id": uuid.uuid4()} credential_info["workspace"] = Workspace.objects.get(id=workspace.id) credential_info["connector_config"] = creds diff --git a/core/social_auth.py b/core/social_auth.py index 3040628..aa57752 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -53,7 +53,6 @@ def login(request, payload: SocialAuthLoginSchema): workspace = Workspace(name="Default Workspace", id=uuid.uuid4(), organization=org) workspace.save() user.save() - warehouse_credentials.DefaultWarehouse.create(workspace) user.organizations.add(org) oauth = OAuthApiKeys(workspace=workspace, type='GOOGLE_LOGIN', id=uuid.uuid4()) oauth.oauth_config = { diff --git a/init_db/connector_def.json b/init_db/connector_def.json index fd08d88..e6d40f4 100644 --- a/init_db/connector_def.json +++ b/init_db/connector_def.json @@ -24,7 +24,7 @@ "type": "DEST", "unique_name": "POSTGRES-DEST", "docker_image": "valmiio/destination-postgres", - "docker_tag": "latest", + "docker_tag": "dev", "display_name": "Dest Postgres", "oauth": "False", "oauth_keys":"private", From 2d97811c0d3402f027d43ba145f8c4772ccb5910 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 7 May 2024 17:46:58 +0530 Subject: [PATCH 057/159] feat: refactored explore creation --- core/explore_api.py | 212 ++++++------------- core/{ => services}/destination_catalog.json | 0 core/services/explore_service.py | 167 +++++++++++++++ core/{ => services}/source_catalog.json | 0 4 files changed, 234 insertions(+), 145 deletions(-) rename core/{ => services}/destination_catalog.json (100%) create mode 100644 core/services/explore_service.py rename core/{ => services}/source_catalog.json (100%) diff --git a/core/explore_api.py b/core/explore_api.py index cd72782..c49d6aa 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,11 +1,8 @@ import datetime import json import logging -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build import json import os -from os.path import dirname, join from typing import List import uuid from decouple import config @@ -14,14 +11,12 @@ from pydantic import Json import requests -from core.api import create_new_run -from .models import OAuthApiKeys, Source import psycopg2 -from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync +from core.models import Account, Credential, Explore, Prompt, StorageCredentials, Workspace from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router - +from core.services.explore_service import ExploreService logger = logging.getLogger(__name__) router = Router() @@ -40,12 +35,7 @@ def get_explores(request,workspace_id): @router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) def create_explore(request, workspace_id,payload: ExploreSchemaIn): - logger.info("logging rquest") - logger.info(request.user) - logger.info("data before creating") data = payload.dict() - logger.info("data before creating") - logger.info(data) try: data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) @@ -58,102 +48,29 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): account_info["workspace"] = data["workspace"] account = Account.objects.create(**account_info) data["account"] = account - logger.debug(data) - oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") - src_credential = {"id": uuid.uuid4()} - src_credential["workspace"] = Workspace.objects.get(id=workspace_id) - src_credential["connector_id"] = "SRC_POSTGRES" - src_credential["name"] = "SRC_POSTGRES 2819" - src_credential["account"] = account - src_credential["status"] = "active" - #TODO: give nice nmaes for each creation(sync,credentials,sources,destination) - storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) - src_connector_config = { - "ssl": False, - "host": "classspace.in", - "port": 5432, - "user": storage_credential.connector_config["username"], - "database": "dvdrental", - "password": storage_credential.connector_config["password"], - "namespace": storage_credential.connector_config["namespace"], - } - src_credential["connector_config"] = src_connector_config - src_cred = Credential.objects.create(**src_credential) - logger.info(src_cred) - des_credential = {"id": uuid.uuid4()} - des_credential["workspace"] = Workspace.objects.get(id=workspace_id) - des_credential["connector_id"] = "DEST_GOOGLE-SHEETS" - des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" - des_credential["account"] = account - des_credential["status"] = "active" - name = f"valmi.io {prompt.name}" - spreadsheet_url = create_spreadsheet(name,refresh_token=oauthkeys.oauth_config["refresh_token"]) - des_connector_config = { - "spreadsheet_id": spreadsheet_url, - "credentials": { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": oauthkeys.oauth_config["refresh_token"], - }, - } - des_credential["connector_config"] = des_connector_config - des_cred = Credential.objects.create(**des_credential) - source_connector = { - "name":"SRC_POSTGRES 2819", - "id":uuid.uuid4() - } - source_connector["workspace"] = Workspace.objects.get(id=workspace_id) - source_connector["credential"] = Credential.objects.get(id=src_cred.id) - source_connector_catalog = {} - json_file_path = join(dirname(__file__), 'source_catalog.json') - with open(json_file_path, 'r') as openfile: - source_connector_catalog = json.load(openfile) - source_connector["catalog"] = source_connector_catalog - source_connector["status"] = "active" - src_connector = Source.objects.create(**source_connector) - logger.info(src_connector) - time.sleep(3) - destination_connector = { - "name":"DEST_GOOGLE-SHEETS 2819", - "id":uuid.uuid4() - - } - destination_connector["workspace"] = Workspace.objects.get(id=workspace_id) - destination_connector["credential"] = Credential.objects.get(id=des_cred.id) - destination_connector_catalog = {} - json_file_path = join(dirname(__file__), 'destination_catalog.json') - with open(json_file_path, 'r') as openfile: - destination_connector_catalog = json.load(openfile) - destination_connector["catalog"] = destination_connector_catalog - des_connector = Destination.objects.create(**destination_connector) - logger.info(des_connector) - time.sleep(3) - sync_config = { - "name":"test 2819", - "id":uuid.uuid4(), - "status":"active", - "ui_state":{} - - } - schedule = {"run_interval": 3600000} - sync_config["schedule"] = schedule - sync_config["source"] = Source.objects.get(id=src_connector.id) - sync_config["destination"] = Destination.objects.get(id=des_connector.id) - sync_config["workspace"] = Workspace.objects.get(id=workspace_id) - sync = Sync.objects.create(**sync_config) - logger.info(sync) - time.sleep(10) + #create source + source = ExploreService.create_source(workspace_id,account) + #create destination + spreadsheet_name = f"valmiio {prompt.name} sheet" + destination = ExploreService.create_destination(spreadsheet_name,workspace_id,account) + logger.debug("after service creation") + logger.info(destination.id) + #create sync + sync = ExploreService.create_sync(source,destination,workspace_id) + time.sleep(5) + #create run payload = SyncStartStopSchemaIn(full_refresh=True) - response = create_new_run(request,workspace_id,sync.id,payload) - print(response) - data["name"] = name + ExploreService.create_run(request,workspace_id,sync.id,payload) + #creating explore + data["name"] = f"valmiio {prompt.name}" data["sync"] = sync explore = Explore.objects.create(**data) + spreadsheet_url = Credential.objects.get(id=destination.credential.id).connector_config["spreadsheet_id"] explore.spreadsheet_url = spreadsheet_url explore.save() return explore - except Exception: - logger.exception("explore creation error") + except Exception as e: + logger.exception(e) return (400, {"detail": "The specific explore cannot be created."}) @@ -197,49 +114,54 @@ def get_explores(request,workspace_id,explore_id): logger.exception("explore listing error") return (400, {"detail": "The explore cannot be fetched."}) -def create_spreadsheet(name,refresh_token): - logger.debug("create_spreadsheet") - SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] - credentials_dict = { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": refresh_token - } - sheet_name = f'{name} sheet' - # Create a Credentials object from the dictionary - credentials = Credentials.from_authorized_user_info( - credentials_dict, scopes=SCOPES - ) - service = build("sheets", "v4", credentials=credentials) - - # Create the spreadsheet - spreadsheet = {"properties": {"title": sheet_name}} - try: - spreadsheet = ( - service.spreadsheets() - .create(body=spreadsheet, fields="spreadsheetId") - .execute() - ) - spreadsheet_id = spreadsheet.get("spreadsheetId") - - #Update the sharing settings to make the spreadsheet publicly accessible - drive_service = build('drive', 'v3', credentials=credentials) - drive_service.permissions().create( - fileId=spreadsheet_id, - body={ - "role": "writer", - "type": "anyone", - "withLink": True - }, - fields="id" - ).execute() - - spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" - print(f"Spreadsheet URL: {spreadsheet_url}") - return spreadsheet_url - except Exception as e: - logger.error(f"Error creating spreadsheet: {e}") - return e +# def create_spreadsheet(name,refresh_token): + # try: + # explore.ExploreService.create_spreadsheet(name,refresh_token) + # except Exception as e: + # logger.exception("create_spreadsheet error") + # return (400, {"detail": "The specific explore cannot be created."}) + # logger.debug("create_spreadsheet") + # SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] + # credentials_dict = { + # "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + # "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + # "refresh_token": refresh_token + # } + # sheet_name = f'{name} sheet' + # # Create a Credentials object from the dictionary + # credentials = Credentials.from_authorized_user_info( + # credentials_dict, scopes=SCOPES + # ) + # service = build("sheets", "v4", credentials=credentials) + + # # Create the spreadsheet + # spreadsheet = {"properties": {"title": sheet_name}} + # try: + # spreadsheet = ( + # service.spreadsheets() + # .create(body=spreadsheet, fields="spreadsheetId") + # .execute() + # ) + # spreadsheet_id = spreadsheet.get("spreadsheetId") + + # #Update the sharing settings to make the spreadsheet publicly accessible + # drive_service = build('drive', 'v3', credentials=credentials) + # drive_service.permissions().create( + # fileId=spreadsheet_id, + # body={ + # "role": "writer", + # "type": "anyone", + # "withLink": True + # }, + # fields="id" + # ).execute() + + # spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" + # print(f"Spreadsheet URL: {spreadsheet_url}") + # return spreadsheet_url + # except Exception as e: + # logger.error(f"Error creating spreadsheet: {e}") + # return e @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) diff --git a/core/destination_catalog.json b/core/services/destination_catalog.json similarity index 100% rename from core/destination_catalog.json rename to core/services/destination_catalog.json diff --git a/core/services/explore_service.py b/core/services/explore_service.py new file mode 100644 index 0000000..14595b4 --- /dev/null +++ b/core/services/explore_service.py @@ -0,0 +1,167 @@ +import json +import logging +import os +import uuid +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from os.path import dirname, join +from core.api import create_new_run +from core.models import Credential, Destination, OAuthApiKeys, Source, StorageCredentials, Sync, Workspace +logger = logging.getLogger(__name__) +class ExploreService: + @staticmethod + def create_spreadsheet(name:str,refresh_token:str): + logger.debug("create_spreadsheet") + SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] + credentials_dict = { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": refresh_token + } + # Create a Credentials object from the dictionary + try: + credentials = Credentials.from_authorized_user_info( + credentials_dict, scopes=SCOPES + ) + service = build("sheets", "v4", credentials=credentials) + # Create the spreadsheet + spreadsheet = {"properties": {"title": name}} + spreadsheet = ( + service.spreadsheets() + .create(body=spreadsheet, fields="spreadsheetId") + .execute() + ) + spreadsheet_id = spreadsheet.get("spreadsheetId") + #Update the sharing settings to make the spreadsheet publicly accessible + drive_service = build('drive', 'v3', credentials=credentials) + drive_service.permissions().create( + fileId=spreadsheet_id, + body={ + "role": "writer", + "type": "anyone", + "withLink": True + }, + fields="id" + ).execute() + + spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" + print(f"Spreadsheet URL: {spreadsheet_url}") + return spreadsheet_url + except Exception as e: + logger.error(f"Error creating spreadsheet: {e}") + raise Exception("spreadhseet creationm failed") + + @staticmethod + def create_source(workspace_id,account:object): + try: + #creating source credentail + credential = {"id": uuid.uuid4()} + credential["workspace"] = Workspace.objects.get(id=workspace_id) + credential["connector_id"] = "SRC_POSTGRES" + credential["name"] = "SRC_POSTGRES" + credential["account"] = account + credential["status"] = "active" + storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) + connector_config = { + "ssl": False, + "host": storage_credential.connector_config["host"], + "port": storage_credential.connector_config["port"], + "user": storage_credential.connector_config["username"], + "database": storage_credential.connector_config["database"], + "password": storage_credential.connector_config["password"], + "namespace": storage_credential.connector_config["namespace"], + } + credential["connector_config"] = connector_config + cred = Credential.objects.create(**credential) + source = { + "name":"SRC_POSTGRES", + "id":uuid.uuid4() + } + #creating source object + source["workspace"] = Workspace.objects.get(id=workspace_id) + source["credential"] = Credential.objects.get(id=cred.id) + source_catalog = {} + json_file_path = join(dirname(__file__), 'source_catalog.json') + with open(json_file_path, 'r') as openfile: + source_catalog = json.load(openfile) + source["catalog"] = source_catalog + source["status"] = "active" + result = Source.objects.create(**source) + return result + except Exception as e: + logger.error(f"Error creating source: {e}") + raise Exception("unable to create source") + + @staticmethod + def create_destination(spreadsheet_name:str,workspace_id,account:object): + try: + #creating destination credential + oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") + credential = {"id": uuid.uuid4()} + credential["workspace"] = Workspace.objects.get(id=workspace_id) + credential["connector_id"] = "DEST_GOOGLE-SHEETS" + credential["name"] = "DEST_GOOGLE-SHEETS" + credential["account"] = account + credential["status"] = "active" + spreadsheet_url = ExploreService.create_spreadsheet(spreadsheet_name,refresh_token=oauthkeys.oauth_config["refresh_token"]) + connector_config = { + "spreadsheet_id": spreadsheet_url, + "credentials": { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": oauthkeys.oauth_config["refresh_token"], + }, + } + credential["connector_config"] = connector_config + cred = Credential.objects.create(**credential) + destination= { + "name":"DEST_GOOGLE-SHEETS", + "id":uuid.uuid4() + + } + #creating destination object + destination["workspace"] = Workspace.objects.get(id=workspace_id) + destination["credential"] = Credential.objects.get(id=cred.id) + destination_catalog = {} + json_file_path = join(dirname(__file__), 'destination_catalog.json') + with open(json_file_path, 'r') as openfile: + destination_catalog = json.load(openfile) + destination["catalog"] = destination_catalog + result = Destination.objects.create(**destination) + logger.info(result) + return result + except Exception as e: + logger.error(f"Error creating destination: {e}") + raise Exception("unable to create destination") + + @staticmethod + def create_sync(source:object,destination:object,workspace_id): + try: + logger.debug("creating sync in service") + logger.debug(source.id) + sync_config = { + "name":"Warehouse to sheets", + "id":uuid.uuid4(), + "status":"active", + "ui_state":{} + + } + schedule = {"run_interval": 3600000} + sync_config["schedule"] = schedule + sync_config["source"] = source + sync_config["destination"] = destination + sync_config["workspace"] = Workspace.objects.get(id=workspace_id) + sync = Sync.objects.create(**sync_config) + return sync + except Exception as e: + logger.error(f"Error creating sync: {e}") + raise Exception("unable to create sync") + + @staticmethod + def create_run(request,workspace_id,sync_id,payload): + try: + response = create_new_run(request,workspace_id,sync_id,payload) + logger.debug(response) + except Exception as e: + logger.error(f"Error creating run: {e}") + raise Exception("unable to create run") \ No newline at end of file diff --git a/core/source_catalog.json b/core/services/source_catalog.json similarity index 100% rename from core/source_catalog.json rename to core/services/source_catalog.json From 124e423605b72b5cf913bd7cef47414200525dd8 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 7 May 2024 17:48:09 +0530 Subject: [PATCH 058/159] feat: refactor explore creation --- core/explore_api.py | 49 --------------------------------------------- 1 file changed, 49 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index c49d6aa..92f3c4e 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -114,55 +114,6 @@ def get_explores(request,workspace_id,explore_id): logger.exception("explore listing error") return (400, {"detail": "The explore cannot be fetched."}) -# def create_spreadsheet(name,refresh_token): - # try: - # explore.ExploreService.create_spreadsheet(name,refresh_token) - # except Exception as e: - # logger.exception("create_spreadsheet error") - # return (400, {"detail": "The specific explore cannot be created."}) - # logger.debug("create_spreadsheet") - # SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] - # credentials_dict = { - # "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - # "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - # "refresh_token": refresh_token - # } - # sheet_name = f'{name} sheet' - # # Create a Credentials object from the dictionary - # credentials = Credentials.from_authorized_user_info( - # credentials_dict, scopes=SCOPES - # ) - # service = build("sheets", "v4", credentials=credentials) - - # # Create the spreadsheet - # spreadsheet = {"properties": {"title": sheet_name}} - # try: - # spreadsheet = ( - # service.spreadsheets() - # .create(body=spreadsheet, fields="spreadsheetId") - # .execute() - # ) - # spreadsheet_id = spreadsheet.get("spreadsheetId") - - # #Update the sharing settings to make the spreadsheet publicly accessible - # drive_service = build('drive', 'v3', credentials=credentials) - # drive_service.permissions().create( - # fileId=spreadsheet_id, - # body={ - # "role": "writer", - # "type": "anyone", - # "withLink": True - # }, - # fields="id" - # ).execute() - - # spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" - # print(f"Spreadsheet URL: {spreadsheet_url}") - # return spreadsheet_url - # except Exception as e: - # logger.error(f"Error creating spreadsheet: {e}") - # return e - @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id): From d345482a27ac3c9e0004633561fce54c605564e3 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Wed, 8 May 2024 10:50:47 +0530 Subject: [PATCH 059/159] feat: prompt list changes --- core/api.py | 1 + core/migrations/0027_auto_20240507_1147.py | 22 +++++++++ core/migrations/0028_auto_20240507_1156.py | 18 +++++++ core/models.py | 13 ++--- core/prompt_api.py | 22 ++++++--- core/schemas.py | 9 +++- .../order_with_product.liquid | 0 core/services/prompts.py | 18 +++++++ init_db/prompt_init.py | 6 +-- init_db/test_prompt_def.json | 48 +++++++++++++++++++ 10 files changed, 138 insertions(+), 19 deletions(-) create mode 100644 core/migrations/0027_auto_20240507_1147.py create mode 100644 core/migrations/0028_auto_20240507_1156.py create mode 100644 core/services/prompt_templates/order_with_product.liquid create mode 100644 core/services/prompts.py create mode 100644 init_db/test_prompt_def.json diff --git a/core/api.py b/core/api.py index dff1155..08d3d1c 100644 --- a/core/api.py +++ b/core/api.py @@ -233,6 +233,7 @@ def create_connection_with_default_warehouse(request, workspace_id,payload: Conn @router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) def storage_credentials(request, workspace_id): config={} + logger.info("came here in storeage") try: creds = StorageCredentials.objects.filter(workspace_id=workspace_id).get( connector_config__shopify_store="chitumalla-store" diff --git a/core/migrations/0027_auto_20240507_1147.py b/core/migrations/0027_auto_20240507_1147.py new file mode 100644 index 0000000..c30e7ab --- /dev/null +++ b/core/migrations/0027_auto_20240507_1147.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.5 on 2024-05-07 11:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0026_auto_20240503_0513'), + ] + + operations = [ + migrations.RemoveField( + model_name='prompt', + name='query', + ), + migrations.AddField( + model_name='prompt', + name='type', + field=models.CharField(default='SRC_SHOPIFY', max_length=256), + ), + ] diff --git a/core/migrations/0028_auto_20240507_1156.py b/core/migrations/0028_auto_20240507_1156.py new file mode 100644 index 0000000..b8b5a6c --- /dev/null +++ b/core/migrations/0028_auto_20240507_1156.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-05-07 11:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0027_auto_20240507_1147'), + ] + + operations = [ + migrations.RenameField( + model_name='prompt', + old_name='parameters', + new_name='spec', + ), + ] diff --git a/core/models.py b/core/models.py index 19ef82a..8383f60 100644 --- a/core/models.py +++ b/core/models.py @@ -6,17 +6,14 @@ """ -from django.utils import timezone import uuid +from enum import Enum from django.contrib.auth import get_user_model from django.contrib.auth.models import AbstractUser -from django.db import models from django.contrib.postgres.fields import ArrayField - - -from enum import Enum - +from django.db import models +from django.utils import timezone # Create your models here. @@ -147,8 +144,8 @@ class Prompt(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) name = models.CharField(max_length=256, null=False, blank=False,unique=True) description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") - query = models.CharField(null=False, blank = False,max_length=5000) - parameters = models.JSONField(blank=False, null=True) + type = models.CharField(null=False, blank = False,max_length=256, default="SRC_SHOPIFY") + spec = models.JSONField(blank=False, null=True) table = models.CharField(max_length=256,null=False, blank=False,default="table_name") package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) diff --git a/core/prompt_api.py b/core/prompt_api.py index 008a265..7978fcc 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -1,22 +1,30 @@ import logging from typing import List -from core.models import Prompt -from core.schemas import DetailSchema, PromptSchema +from core.models import Prompt, Credential +from core.schemas import DetailSchema, PromptSchema,PromptSchemaOut from ninja import Router +import json logger = logging.getLogger(__name__) router = Router() -@router.get("/", response={200: List[PromptSchema], 400: DetailSchema}) +@router.get("/", response={200: List[PromptSchemaOut], 400: DetailSchema}) def get_prompts(request): try: - logger.debug("listing prompts") - prompts = Prompt.objects.all() + prompts = list(Prompt.objects.all().values()) + connector_ids = list(Credential.objects.values('connector_id').distinct()) + connector_types = [connector['connector_id'] for connector in connector_ids] + for prompt in prompts: + prompt["id"] = str(prompt["id"]) + if prompt["type"] in connector_types: + prompt["enabled"] = True + else: + prompt["enabled"] = False return prompts - except Exception: - logger.exception("prompts listing error") + except Exception as err: + logger.exception("prompts listing error:"+ err) return (400, {"detail": "The list of prompts cannot be fetched."}) @router.get("/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) diff --git a/core/schemas.py b/core/schemas.py index 387e43e..3045617 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -71,7 +71,14 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","description","query","parameters","package_id","gated","table"] + model_fields = ["id","name","description","type","spec","package_id","gated","table"] + +class PromptSchemaOut(Schema): + id: str + name: str + description: str + type: str + enabled: bool class ExploreStatusIn(Schema): sync_id:str diff --git a/core/services/prompt_templates/order_with_product.liquid b/core/services/prompt_templates/order_with_product.liquid new file mode 100644 index 0000000..e69de29 diff --git a/core/services/prompts.py b/core/services/prompts.py new file mode 100644 index 0000000..fc612ed --- /dev/null +++ b/core/services/prompts.py @@ -0,0 +1,18 @@ +from liquid import Environment, FileSystemLoader + + +class PromptService(): + @classmethod + def getTemplateFile(cls, model:str): + return model + '.liquid' + @staticmethod + def build(model, timeWindow, filters)-> str: + file_name = PromptService.getTemplateFile(model) + env = Environment(loader=FileSystemLoader("templates/")) + template = env.get_template(file_name) + return template.render(model=model, timeWindow=timeWindow, filters=filters) + + + + + \ No newline at end of file diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index ce31d09..ebb128a 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -14,7 +14,7 @@ from requests.auth import HTTPBasicAuth -prompt_defs = json.loads(open(join(dirname(__file__), "prompt_def.json"), "r").read()) +prompt_defs = json.loads(open(join(dirname(__file__), "test_prompt_def.json"), "r").read()) for prompt_def in prompt_defs["definitions"]: resp = requests.post( @@ -23,9 +23,9 @@ "id":str(uuid.uuid4()), "name": prompt_def["name"], "description": prompt_def["description"], - "query": prompt_def["query"], + "type": prompt_def["type"], "table":prompt_def["table"], - "parameters":prompt_def["parameters"], + "spec":prompt_def["spec"], "package_id":prompt_def["package_id"], "gated":prompt_def["gated"], }, diff --git a/init_db/test_prompt_def.json b/init_db/test_prompt_def.json new file mode 100644 index 0000000..b1ea308 --- /dev/null +++ b/init_db/test_prompt_def.json @@ -0,0 +1,48 @@ +{ + "definitions": [ + { + "name": "Inventory snapshot", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "table": "orders_with_product_data", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "anyOf": [ + { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + } + ] + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + } + ] +} From aaccfb35e88c4a43eabd46bc2b450b5cf86d9355 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Wed, 8 May 2024 11:10:50 +0530 Subject: [PATCH 060/159] feat: refactor migrations --- ...0507_1147.py => 0027_auto_20240508_0539.py} | 7 ++++++- core/migrations/0028_auto_20240507_1156.py | 18 ------------------ 2 files changed, 6 insertions(+), 19 deletions(-) rename core/migrations/{0027_auto_20240507_1147.py => 0027_auto_20240508_0539.py} (70%) delete mode 100644 core/migrations/0028_auto_20240507_1156.py diff --git a/core/migrations/0027_auto_20240507_1147.py b/core/migrations/0027_auto_20240508_0539.py similarity index 70% rename from core/migrations/0027_auto_20240507_1147.py rename to core/migrations/0027_auto_20240508_0539.py index c30e7ab..3fc94d5 100644 --- a/core/migrations/0027_auto_20240507_1147.py +++ b/core/migrations/0027_auto_20240508_0539.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.5 on 2024-05-07 11:47 +# Generated by Django 3.1.5 on 2024-05-08 05:39 from django.db import migrations, models @@ -10,6 +10,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RenameField( + model_name='prompt', + old_name='parameters', + new_name='spec', + ), migrations.RemoveField( model_name='prompt', name='query', diff --git a/core/migrations/0028_auto_20240507_1156.py b/core/migrations/0028_auto_20240507_1156.py deleted file mode 100644 index b8b5a6c..0000000 --- a/core/migrations/0028_auto_20240507_1156.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.5 on 2024-05-07 11:56 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0027_auto_20240507_1147'), - ] - - operations = [ - migrations.RenameField( - model_name='prompt', - old_name='parameters', - new_name='spec', - ), - ] From 5c15e38f597de066755cc0d1cb0647a94c5c67dc Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 8 May 2024 11:45:36 +0530 Subject: [PATCH 061/159] feat: refactor explore creation --- core/explore_api.py | 16 +++++++-------- core/services/explore_service.py | 34 +++++++++++++++++--------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 92f3c4e..60c3134 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -13,7 +13,7 @@ import psycopg2 -from core.models import Account, Credential, Explore, Prompt, StorageCredentials, Workspace +from core.models import Account, Explore, Prompt, StorageCredentials, Workspace from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router from core.services.explore_service import ExploreService @@ -52,22 +52,22 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): source = ExploreService.create_source(workspace_id,account) #create destination spreadsheet_name = f"valmiio {prompt.name} sheet" - destination = ExploreService.create_destination(spreadsheet_name,workspace_id,account) + destination_data = ExploreService.create_destination(spreadsheet_name,workspace_id,account) + spreadsheet_url = destination_data[0] + destination = destination_data[1] logger.debug("after service creation") logger.info(destination.id) #create sync sync = ExploreService.create_sync(source,destination,workspace_id) time.sleep(5) - #create run - payload = SyncStartStopSchemaIn(full_refresh=True) - ExploreService.create_run(request,workspace_id,sync.id,payload) #creating explore data["name"] = f"valmiio {prompt.name}" data["sync"] = sync + data["spreadsheet_url"] = spreadsheet_url explore = Explore.objects.create(**data) - spreadsheet_url = Credential.objects.get(id=destination.credential.id).connector_config["spreadsheet_id"] - explore.spreadsheet_url = spreadsheet_url - explore.save() + #create run + payload = SyncStartStopSchemaIn(full_refresh=True) + ExploreService.create_run(request,workspace_id,sync.id,payload) return explore except Exception as e: logger.exception(e) diff --git a/core/services/explore_service.py b/core/services/explore_service.py index 14595b4..48c1ca0 100644 --- a/core/services/explore_service.py +++ b/core/services/explore_service.py @@ -1,6 +1,7 @@ import json import logging import os +from typing import List, Union import uuid from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -8,11 +9,12 @@ from core.api import create_new_run from core.models import Credential, Destination, OAuthApiKeys, Source, StorageCredentials, Sync, Workspace logger = logging.getLogger(__name__) + +SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] class ExploreService: @staticmethod - def create_spreadsheet(name:str,refresh_token:str): + def create_spreadsheet(name:str,refresh_token:str)->str: logger.debug("create_spreadsheet") - SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] credentials_dict = { "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], @@ -20,8 +22,9 @@ def create_spreadsheet(name:str,refresh_token:str): } # Create a Credentials object from the dictionary try: + base_spreadsheet_url = "https://docs.google.com/spreadsheets/d/" credentials = Credentials.from_authorized_user_info( - credentials_dict, scopes=SCOPES + credentials_dict, scopes=SPREADSHEET_SCOPES ) service = build("sheets", "v4", credentials=credentials) # Create the spreadsheet @@ -44,15 +47,14 @@ def create_spreadsheet(name:str,refresh_token:str): fields="id" ).execute() - spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" - print(f"Spreadsheet URL: {spreadsheet_url}") + spreadsheet_url = f"{base_spreadsheet_url}{spreadsheet_id}" return spreadsheet_url except Exception as e: - logger.error(f"Error creating spreadsheet: {e}") - raise Exception("spreadhseet creationm failed") + logger.exception(f"Error creating spreadsheet: {e}") + raise Exception("spreadhseet creation failed") @staticmethod - def create_source(workspace_id,account:object): + def create_source(workspace_id:str,account:object)->object: try: #creating source credentail credential = {"id": uuid.uuid4()} @@ -89,11 +91,11 @@ def create_source(workspace_id,account:object): result = Source.objects.create(**source) return result except Exception as e: - logger.error(f"Error creating source: {e}") + logger.exception(f"Error creating source: {e}") raise Exception("unable to create source") @staticmethod - def create_destination(spreadsheet_name:str,workspace_id,account:object): + def create_destination(spreadsheet_name:str,workspace_id:str,account:object)->List[Union[str, object]]: try: #creating destination credential oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") @@ -129,13 +131,13 @@ def create_destination(spreadsheet_name:str,workspace_id,account:object): destination["catalog"] = destination_catalog result = Destination.objects.create(**destination) logger.info(result) - return result + return [spreadsheet_url,result] except Exception as e: - logger.error(f"Error creating destination: {e}") + logger.exception(f"Error creating destination: {e}") raise Exception("unable to create destination") @staticmethod - def create_sync(source:object,destination:object,workspace_id): + def create_sync(source:object,destination:object,workspace_id:str)->object: try: logger.debug("creating sync in service") logger.debug(source.id) @@ -154,14 +156,14 @@ def create_sync(source:object,destination:object,workspace_id): sync = Sync.objects.create(**sync_config) return sync except Exception as e: - logger.error(f"Error creating sync: {e}") + logger.exception(f"Error creating sync: {e}") raise Exception("unable to create sync") @staticmethod - def create_run(request,workspace_id,sync_id,payload): + def create_run(request:object,workspace_id:str,sync_id:str,payload:object)->None: try: response = create_new_run(request,workspace_id,sync_id,payload) logger.debug(response) except Exception as e: - logger.error(f"Error creating run: {e}") + logger.exception(f"Error creating run: {e}") raise Exception("unable to create run") \ No newline at end of file From 12c63d427056a1ad9c70e1d533d3f5ed134d0252 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Wed, 8 May 2024 12:56:20 +0530 Subject: [PATCH 062/159] feat: schemas refactor --- core/api.py | 2 +- core/engine_api.py | 2 +- core/explore_api.py | 35 +------------------ core/oauth_api.py | 2 +- core/package_api.py | 2 +- core/prompt_api.py | 33 ++++++++++++++--- core/schemas/__init__.py | 0 core/schemas/prompt.py | 25 +++++++++++++ core/{ => schemas}/schemas.py | 2 +- core/services/__init__.py | 0 .../order_with_product.liquid | 9 +++++ core/services/prompts.py | 10 +++--- core/social_auth.py | 20 ++++++----- core/stream_api.py | 2 +- requirements.txt | 3 +- 15 files changed, 88 insertions(+), 59 deletions(-) create mode 100644 core/schemas/__init__.py create mode 100644 core/schemas/prompt.py rename core/{ => schemas}/schemas.py (97%) create mode 100644 core/services/__init__.py diff --git a/core/api.py b/core/api.py index 08d3d1c..866bba9 100644 --- a/core/api.py +++ b/core/api.py @@ -16,7 +16,7 @@ from decouple import Csv, config from ninja import Router from pydantic import UUID4, Json -from core.schemas import ( +from core.schemas.schemas import ( ConnectionSchemaIn, ConnectorConfigSchemaIn, ConnectorSchema, diff --git a/core/engine_api.py b/core/engine_api.py index a18d61a..d7dfb89 100644 --- a/core/engine_api.py +++ b/core/engine_api.py @@ -12,7 +12,7 @@ from decouple import Csv, config from ninja import Router -from core.schemas import ConnectorSchema, DetailSchema, PackageSchema, PromptSchema, SyncSchema +from core.schemas.schemas import ConnectorSchema, DetailSchema, PackageSchema, PromptSchema, SyncSchema from .models import ( Connector, diff --git a/core/explore_api.py b/core/explore_api.py index cd72782..2d6698d 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,4 +1,3 @@ -import datetime import json import logging from google.oauth2.credentials import Credentials @@ -19,7 +18,7 @@ from .models import OAuthApiKeys, Source import psycopg2 from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn +from core.schemas.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router logger = logging.getLogger(__name__) @@ -156,38 +155,6 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): logger.exception("explore creation error") return (400, {"detail": "The specific explore cannot be created."}) - -def custom_serializer(obj): - if isinstance(obj, datetime.datetime): - return obj.isoformat() - -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) -def preview_data(request, workspace_id,prompt_id): - prompt = Prompt.objects.get(id=prompt_id) - storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) - host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = storage_cred.connector_config.get('password') - db_username = storage_cred.connector_config.get('username') - db_namespace = storage_cred.connector_config.get('namespace') - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - table = prompt.table - # LOGIC for implementing pagination if necessary - # query = f'SELECT COUNT(*) FROM {db_namespace}.{table}' - # cursor.execute(query) - # count_row = cursor.fetchone() - # count = count_row[0] if count_row is not None else 0 - # page_id_int = int(page_id) - - # if page_id_int > count/25 or page_id_int<=0: - # return (404, {"detail": "Invalid page Id."}) - # skip = 25*(page_id_int-1) - query = f'SELECT * FROM {db_namespace}.{table} LIMIT 100' - cursor.execute(query) - items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] - return json.dumps(items, indent=4, default=custom_serializer) - - @router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) def get_explores(request,workspace_id,explore_id): try: diff --git a/core/oauth_api.py b/core/oauth_api.py index 84cc813..8dfe0ed 100644 --- a/core/oauth_api.py +++ b/core/oauth_api.py @@ -18,7 +18,7 @@ from .models import Connector, Workspace, OAuthApiKeys -from core.schemas import ( +from core.schemas.schemas import ( DetailSchema, OAuthSchema, OAuthSchemaIn, diff --git a/core/package_api.py b/core/package_api.py index 661ff63..3abd862 100644 --- a/core/package_api.py +++ b/core/package_api.py @@ -2,7 +2,7 @@ from typing import List from core.models import Package -from core.schemas import DetailSchema, PackageSchema +from core.schemas.schemas import DetailSchema, PackageSchema from ninja import Router logger = logging.getLogger(__name__) diff --git a/core/prompt_api.py b/core/prompt_api.py index 7978fcc..f3bb7ab 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -1,10 +1,17 @@ +import datetime +import json import logging +import os from typing import List -from core.models import Prompt, Credential -from core.schemas import DetailSchema, PromptSchema,PromptSchemaOut +import psycopg2 from ninja import Router -import json +from pydantic import Json + +from core.models import Credential, Prompt, StorageCredentials +from core.schemas.prompt import PromptPreviewSchemaIn +from core.schemas.schemas import DetailSchema, PromptSchema, PromptSchemaOut +from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -37,4 +44,22 @@ def get_prompts(request,prompt_id): logger.exception("prompt listing error") return (400, {"detail": "The prompt cannot be fetched."}) - \ No newline at end of file + +def custom_serializer(obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) +def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): + prompt = Prompt.objects.get(id=prompt_id) + query = PromptService().build(prompt.table, prompt_req.time_window, prompt_req.filters) + storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = storage_cred.connector_config.get('password') + db_username = storage_cred.connector_config.get('username') + db_namespace = storage_cred.connector_config.get('namespace') + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + cursor.execute(query) + items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] + return json.dumps(items, indent=4, default=custom_serializer) diff --git a/core/schemas/__init__.py b/core/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py new file mode 100644 index 0000000..db280a1 --- /dev/null +++ b/core/schemas/prompt.py @@ -0,0 +1,25 @@ +from ninja import Schema + + +class TimeWindowRange(Schema): + start: str + end: str + +class TimeWindow(Schema): + label: str + range: TimeWindowRange + + +class Filter(Schema): + label: str + operator: str + name: str + value: str + + +class PromptPreviewSchemaIn(Schema): + source_id: str + time_window: TimeWindow + filters: list[Filter] + + diff --git a/core/schemas.py b/core/schemas/schemas.py similarity index 97% rename from core/schemas.py rename to core/schemas/schemas.py index 3045617..d0ba84d 100644 --- a/core/schemas.py +++ b/core/schemas/schemas.py @@ -12,7 +12,7 @@ from django.contrib.auth import get_user_model from ninja import Field, ModelSchema, Schema from pydantic import UUID4 -from .models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys +from core.models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys User = get_user_model() diff --git a/core/services/__init__.py b/core/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/services/prompt_templates/order_with_product.liquid b/core/services/prompt_templates/order_with_product.liquid index e69de29..45bba88 100644 --- a/core/services/prompt_templates/order_with_product.liquid +++ b/core/services/prompt_templates/order_with_product.liquid @@ -0,0 +1,9 @@ + +{% assign filterStr = "" %} +{% for filter in filters %} + {% assign filterStr = filterStr| append: filter.name | append: " " | append: filter.operator | append: " "| append: filter.value %} + {% if forloop.last == false %} + {% assign filterStr = filterStr| append: " and " %} + {% endif %} +{% endfor %} +select * from {{ table }} where {{ filterStr }} and updated_at >= {{ timeWindow.range.start}} and updated_at <= {{ timeWindow.range.end }} \ No newline at end of file diff --git a/core/services/prompts.py b/core/services/prompts.py index fc612ed..4f56ea4 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -3,14 +3,14 @@ class PromptService(): @classmethod - def getTemplateFile(cls, model:str): - return model + '.liquid' + def getTemplateFile(cls, tab:str): + return 'prompts.liquid' @staticmethod - def build(model, timeWindow, filters)-> str: - file_name = PromptService.getTemplateFile(model) + def build(table, timeWindow, filters)-> str: + file_name = PromptService.getTemplateFile() env = Environment(loader=FileSystemLoader("templates/")) template = env.get_template(file_name) - return template.render(model=model, timeWindow=timeWindow, filters=filters) + return template.render(table=table, timeWindow=timeWindow, filters=filters) diff --git a/core/social_auth.py b/core/social_auth.py index aa57752..b95ea30 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -1,17 +1,19 @@ +import binascii +import json +import logging +import os +import uuid from urllib.parse import urlparse -from ninja import Router, Schema + import psycopg2 +from ninja import Router, Schema from pydantic import Json from rest_framework.authtoken.models import Token -from core.schemas import DetailSchema, SocialAuthLoginSchema -from core.models import User, Organization, Workspace, OAuthApiKeys -from core.services import warehouse_credentials -import binascii -import os -import uuid -import logging -import json + +from core.models import OAuthApiKeys, Organization, User, Workspace +from core.schemas.schemas import DetailSchema, SocialAuthLoginSchema + router = Router() diff --git a/core/stream_api.py b/core/stream_api.py index f9d8724..ff3d3ed 100644 --- a/core/stream_api.py +++ b/core/stream_api.py @@ -14,7 +14,7 @@ from pydantic import Json from decouple import config from core.api import SHORT_TIMEOUT -from core.schemas import GenericJsonSchema +from core.schemas.schemas import GenericJsonSchema from .models import ValmiUserIDJitsuApiToken from typing import Optional diff --git a/requirements.txt b/requirements.txt index 453d54f..f9a7cc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,5 @@ opentelemetry-exporter-otlp opentelemetry-instrumentation-logging requests google.oauth -google-api-python-client \ No newline at end of file +google-api-python-client +python-liquid==1.12.1 \ No newline at end of file From bdc6e5cdc182c55f8cbfc826e0a822dbec1eb5e7 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 8 May 2024 13:30:55 +0530 Subject: [PATCH 063/159] feat: Create table SourceAccessInfo and update Shopify source creation --- core/api.py | 18 ++++---- core/migrations/0028_sourceaccessinfo.py | 22 ++++++++++ core/models.py | 4 ++ core/prompt_api.py | 8 ++-- core/services/warehouse_credentials.py | 54 +++++++++++++----------- 5 files changed, 69 insertions(+), 37 deletions(-) create mode 100644 core/migrations/0028_sourceaccessinfo.py diff --git a/core/api.py b/core/api.py index 866bba9..5c392d2 100644 --- a/core/api.py +++ b/core/api.py @@ -38,10 +38,9 @@ UserSchemaOut, CreateConfigSchemaIn ) -from .models import Account, Connector, Credential, Destination, Source, StorageCredentials, Sync, User, Workspace, OAuthApiKeys +from core.services.warehouse_credentials import DefaultWarehouse +from .models import Account, Connector, Credential, Destination, Source, SourceAccessInfo, StorageCredentials, Sync, User, Workspace, OAuthApiKeys from valmi_app_backend.utils import replace_values_in_json -from core.services import warehouse_credentials - router = Router() CONNECTOR_PREFIX_URL = config("ACTIVATION_SERVER") + "/connectors" @@ -209,15 +208,14 @@ def create_connection_with_default_warehouse(request, workspace_id,payload: Conn try: source_credential_payload = CredentialSchemaIn(name=data["shopify_store"],account=data["account"],connector_type=data["source_connector_type"],connector_config=data["source_connector_config"]) source_credential = create_credential(request,workspace_id,source_credential_payload) + source_payload = SourceSchemaIn(name="shopify",credential_id=source_credential.id,catalog = data["source_catalog"]) + source = create_source(request,workspace_id,source_payload) workspace = Workspace.objects.get(id = workspace_id) - warehouse_credentials.DefaultWarehouse.create(workspace,data["shopify_store"]) - storage_credentials = StorageCredentials.objects.filter(workspace_id=workspace_id).get( - connector_config__shopify_store=data["shopify_store"] - ) + storage_credentials = DefaultWarehouse.create(workspace) + source_access_info = {"source":source, "storage_credentials":storage_credentials} + SourceAccessInfo.objects.create(**source_access_info) destination_credential_payload = CredentialSchemaIn(name="default warehouse",account=data["account"],connector_type="DEST_POSTGRES-DEST",connector_config=storage_credentials.connector_config) destination_credential = create_credential(request,workspace_id,destination_credential_payload) - source_payload = SourceSchemaIn(name="shopify",credential_id=source_credential.id,catalog = data["source_catalog"]) - source = create_source(request,workspace_id,source_payload) destination_payload = DestinationSchemaIn(name="default warehouse",credential_id=destination_credential.id,catalog = data["destination_catalog"]) destination = create_destination(request,workspace_id,destination_payload) sync_payload = SyncSchemaIn(name="shopify to default warehouse",source_id=source.id,destination_id=destination.id,schedule=data["schedule"]) @@ -227,7 +225,7 @@ def create_connection_with_default_warehouse(request, workspace_id,payload: Conn logger.debug(response) return "starting sync from shopify to default warehouse" except Exception as e: - logger.debug(e.message) + logger.exception(e) return {"detail": "The specific connection cannot be created."} @router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) diff --git a/core/migrations/0028_sourceaccessinfo.py b/core/migrations/0028_sourceaccessinfo.py new file mode 100644 index 0000000..7706c58 --- /dev/null +++ b/core/migrations/0028_sourceaccessinfo.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.5 on 2024-05-08 06:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0027_auto_20240508_0539'), + ] + + operations = [ + migrations.CreateModel( + name='SourceAccessInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='source_access_info', to='core.source')), + ('storage_credentials', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='source_access_info', to='core.storagecredentials')), + ], + ), + ] diff --git a/core/models.py b/core/models.py index 8383f60..2b3f5d4 100644 --- a/core/models.py +++ b/core/models.py @@ -150,6 +150,10 @@ class Prompt(models.Model): package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) +class SourceAccessInfo(models.Model): + source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info") + storage_credentials = models.ForeignKey(to=StorageCredentials,on_delete=models.CASCADE,related_name="source_access_info") + class Account(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) name = models.CharField(max_length=256, null=False, blank=False) diff --git a/core/prompt_api.py b/core/prompt_api.py index f3bb7ab..ab4de46 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -8,7 +8,7 @@ from ninja import Router from pydantic import Json -from core.models import Credential, Prompt, StorageCredentials +from core.models import Credential, Prompt, SourceAccessInfo, StorageCredentials from core.schemas.prompt import PromptPreviewSchemaIn from core.schemas.schemas import DetailSchema, PromptSchema, PromptSchemaOut from core.services.prompts import PromptService @@ -53,13 +53,15 @@ def custom_serializer(obj): def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): prompt = Prompt.objects.get(id=prompt_id) query = PromptService().build(prompt.table, prompt_req.time_window, prompt_req.filters) - storage_cred = StorageCredentials.objects.get(workspace_id=workspace_id) + souce_access_info = SourceAccessInfo.objects.get(id=prompt_req.source_id) + storage_cred = StorageCredentials.objects.get(id=souce_access_info.storage_credentials_id) host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = storage_cred.connector_config.get('password') db_username = storage_cred.connector_config.get('username') - db_namespace = storage_cred.connector_config.get('namespace') conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) cursor = conn.cursor() cursor.execute(query) + conn.commit() + conn.close() items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] return json.dumps(items, indent=4, default=custom_serializer) diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py index 40619d8..c4ff484 100644 --- a/core/services/warehouse_credentials.py +++ b/core/services/warehouse_credentials.py @@ -11,27 +11,33 @@ class DefaultWarehouse(): @staticmethod - def create(workspace,shopify_store): - host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] - db_username = os.environ["DATA_WAREHOUSE_USERNAME"] - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - logger.debug("logger in creating new creds") - user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) - password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name,'shopify_store':shopify_store,'host':'classspace.in','database':'dvdrental','port':5432} - credential_info = {"id": uuid.uuid4()} - credential_info["workspace"] = Workspace.objects.get(id=workspace.id) - credential_info["connector_config"] = creds - result = StorageCredentials.objects.create(**credential_info) - query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) - cursor.execute(query, (password,)) - query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) - cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) - cursor.execute(query) - query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) - cursor.execute(query) - conn.commit() - conn.close() \ No newline at end of file + def create(workspace:object)->object: + try: + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] + db_username = os.environ["DATA_WAREHOUSE_USERNAME"] + conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + cursor = conn.cursor() + logger.debug("logger in creating new creds") + user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) + password = ''.join(random.choices(string.ascii_uppercase, k=17)) + creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name,'host':'classspace.in','database':'dvdrental','port':5432} + credential_info = {"id": uuid.uuid4()} + credential_info["workspace"] = Workspace.objects.get(id=workspace.id) + credential_info["connector_config"] = creds + result = StorageCredentials.objects.create(**credential_info) + query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) + cursor.execute(query, (password,)) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + cursor.execute(query) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) + cursor.execute(query) + query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) + cursor.execute(query) + conn.commit() + conn.close() + return result + except Exception as e: + logger.exception(e) + raise Exception("Could not create warehouse credentials") + \ No newline at end of file From ab146285f3d729391c4e5847cd47f116341b2ecf Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 8 May 2024 14:16:12 +0530 Subject: [PATCH 064/159] feat: Implement explore creation logic using SourceAccessInfo table --- core/explore_api.py | 175 +++---------------- core/migrations/0029_auto_20240508_0844.py | 23 +++ core/models.py | 2 +- core/prompt_api.py | 2 +- core/schemas/schemas.py | 1 + core/{ => services}/destination_catalog.json | 0 core/services/explore_service.py | 170 ++++++++++++++++++ core/{ => services}/source_catalog.json | 0 8 files changed, 220 insertions(+), 153 deletions(-) create mode 100644 core/migrations/0029_auto_20240508_0844.py rename core/{ => services}/destination_catalog.json (100%) create mode 100644 core/services/explore_service.py rename core/{ => services}/source_catalog.json (100%) diff --git a/core/explore_api.py b/core/explore_api.py index 2d6698d..249e6c1 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,10 +1,6 @@ import json import logging -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build import json -import os -from os.path import dirname, join from typing import List import uuid from decouple import config @@ -13,14 +9,11 @@ from pydantic import Json import requests -from core.api import create_new_run -from .models import OAuthApiKeys, Source -import psycopg2 -from core.models import Account, Credential, Destination, Explore, Prompt, StorageCredentials, Workspace,Sync +from core.models import Account, Explore, Prompt, Workspace from core.schemas.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router - +from core.services.explore_service import ExploreService logger = logging.getLogger(__name__) router = Router() @@ -39,12 +32,7 @@ def get_explores(request,workspace_id): @router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) def create_explore(request, workspace_id,payload: ExploreSchemaIn): - logger.info("logging rquest") - logger.info(request.user) - logger.info("data before creating") data = payload.dict() - logger.info("data before creating") - logger.info(data) try: data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) @@ -57,102 +45,31 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): account_info["workspace"] = data["workspace"] account = Account.objects.create(**account_info) data["account"] = account - logger.debug(data) - oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") - src_credential = {"id": uuid.uuid4()} - src_credential["workspace"] = Workspace.objects.get(id=workspace_id) - src_credential["connector_id"] = "SRC_POSTGRES" - src_credential["name"] = "SRC_POSTGRES 2819" - src_credential["account"] = account - src_credential["status"] = "active" - #TODO: give nice nmaes for each creation(sync,credentials,sources,destination) - storage_credential = StorageCredentials.objects.get(workspace_id = workspace_id) - src_connector_config = { - "ssl": False, - "host": "classspace.in", - "port": 5432, - "user": storage_credential.connector_config["username"], - "database": "dvdrental", - "password": storage_credential.connector_config["password"], - "namespace": storage_credential.connector_config["namespace"], - } - src_credential["connector_config"] = src_connector_config - src_cred = Credential.objects.create(**src_credential) - logger.info(src_cred) - des_credential = {"id": uuid.uuid4()} - des_credential["workspace"] = Workspace.objects.get(id=workspace_id) - des_credential["connector_id"] = "DEST_GOOGLE-SHEETS" - des_credential["name"] = "DEST_GOOGLE-SHEETS 2819" - des_credential["account"] = account - des_credential["status"] = "active" - name = f"valmi.io {prompt.name}" - spreadsheet_url = create_spreadsheet(name,refresh_token=oauthkeys.oauth_config["refresh_token"]) - des_connector_config = { - "spreadsheet_id": spreadsheet_url, - "credentials": { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": oauthkeys.oauth_config["refresh_token"], - }, - } - des_credential["connector_config"] = des_connector_config - des_cred = Credential.objects.create(**des_credential) - source_connector = { - "name":"SRC_POSTGRES 2819", - "id":uuid.uuid4() - } - source_connector["workspace"] = Workspace.objects.get(id=workspace_id) - source_connector["credential"] = Credential.objects.get(id=src_cred.id) - source_connector_catalog = {} - json_file_path = join(dirname(__file__), 'source_catalog.json') - with open(json_file_path, 'r') as openfile: - source_connector_catalog = json.load(openfile) - source_connector["catalog"] = source_connector_catalog - source_connector["status"] = "active" - src_connector = Source.objects.create(**source_connector) - logger.info(src_connector) - time.sleep(3) - destination_connector = { - "name":"DEST_GOOGLE-SHEETS 2819", - "id":uuid.uuid4() - - } - destination_connector["workspace"] = Workspace.objects.get(id=workspace_id) - destination_connector["credential"] = Credential.objects.get(id=des_cred.id) - destination_connector_catalog = {} - json_file_path = join(dirname(__file__), 'destination_catalog.json') - with open(json_file_path, 'r') as openfile: - destination_connector_catalog = json.load(openfile) - destination_connector["catalog"] = destination_connector_catalog - des_connector = Destination.objects.create(**destination_connector) - logger.info(des_connector) - time.sleep(3) - sync_config = { - "name":"test 2819", - "id":uuid.uuid4(), - "status":"active", - "ui_state":{} - - } - schedule = {"run_interval": 3600000} - sync_config["schedule"] = schedule - sync_config["source"] = Source.objects.get(id=src_connector.id) - sync_config["destination"] = Destination.objects.get(id=des_connector.id) - sync_config["workspace"] = Workspace.objects.get(id=workspace_id) - sync = Sync.objects.create(**sync_config) - logger.info(sync) - time.sleep(10) - payload = SyncStartStopSchemaIn(full_refresh=True) - response = create_new_run(request,workspace_id,sync.id,payload) - print(response) - data["name"] = name + #create source + logger.debug("source_id is %s",data["source_id"]) + source = ExploreService.create_source(data["source_id"],workspace_id,account) + #create destination + spreadsheet_name = f"valmiio {prompt.name} sheet" + destination_data = ExploreService.create_destination(spreadsheet_name,workspace_id,account) + spreadsheet_url = destination_data[0] + destination = destination_data[1] + logger.debug("after service creation") + logger.info(destination.id) + #create sync + sync = ExploreService.create_sync(source,destination,workspace_id) + time.sleep(5) + #creating explore + data.pop("source_id") + data["name"] = f"valmiio {prompt.name}" data["sync"] = sync + data["spreadsheet_url"] = spreadsheet_url explore = Explore.objects.create(**data) - explore.spreadsheet_url = spreadsheet_url - explore.save() + #create run + payload = SyncStartStopSchemaIn(full_refresh=True) + ExploreService.create_run(request,workspace_id,sync.id,payload) return explore - except Exception: - logger.exception("explore creation error") + except Exception as e: + logger.exception(e) return (400, {"detail": "The specific explore cannot be created."}) @router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) @@ -164,50 +81,6 @@ def get_explores(request,workspace_id,explore_id): logger.exception("explore listing error") return (400, {"detail": "The explore cannot be fetched."}) -def create_spreadsheet(name,refresh_token): - logger.debug("create_spreadsheet") - SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] - credentials_dict = { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], - "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], - "refresh_token": refresh_token - } - sheet_name = f'{name} sheet' - # Create a Credentials object from the dictionary - credentials = Credentials.from_authorized_user_info( - credentials_dict, scopes=SCOPES - ) - service = build("sheets", "v4", credentials=credentials) - - # Create the spreadsheet - spreadsheet = {"properties": {"title": sheet_name}} - try: - spreadsheet = ( - service.spreadsheets() - .create(body=spreadsheet, fields="spreadsheetId") - .execute() - ) - spreadsheet_id = spreadsheet.get("spreadsheetId") - - #Update the sharing settings to make the spreadsheet publicly accessible - drive_service = build('drive', 'v3', credentials=credentials) - drive_service.permissions().create( - fileId=spreadsheet_id, - body={ - "role": "writer", - "type": "anyone", - "withLink": True - }, - fields="id" - ).execute() - - spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{spreadsheet_id}" - print(f"Spreadsheet URL: {spreadsheet_url}") - return spreadsheet_url - except Exception as e: - logger.error(f"Error creating spreadsheet: {e}") - return e - @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id): diff --git a/core/migrations/0029_auto_20240508_0844.py b/core/migrations/0029_auto_20240508_0844.py new file mode 100644 index 0000000..2cb9049 --- /dev/null +++ b/core/migrations/0029_auto_20240508_0844.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.5 on 2024-05-08 08:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0028_sourceaccessinfo'), + ] + + operations = [ + migrations.RemoveField( + model_name='sourceaccessinfo', + name='id', + ), + migrations.AlterField( + model_name='sourceaccessinfo', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='source_access_info', serialize=False, to='core.source'), + ), + ] diff --git a/core/models.py b/core/models.py index 2b3f5d4..a468475 100644 --- a/core/models.py +++ b/core/models.py @@ -151,7 +151,7 @@ class Prompt(models.Model): gated = models.BooleanField(null=False, blank = False, default=True) class SourceAccessInfo(models.Model): - source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info") + source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info",primary_key=True) storage_credentials = models.ForeignKey(to=StorageCredentials,on_delete=models.CASCADE,related_name="source_access_info") class Account(models.Model): diff --git a/core/prompt_api.py b/core/prompt_api.py index ab4de46..36d61a0 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -53,7 +53,7 @@ def custom_serializer(obj): def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): prompt = Prompt.objects.get(id=prompt_id) query = PromptService().build(prompt.table, prompt_req.time_window, prompt_req.filters) - souce_access_info = SourceAccessInfo.objects.get(id=prompt_req.source_id) + souce_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) storage_cred = StorageCredentials.objects.get(id=souce_access_info.storage_credentials_id) host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = storage_cred.connector_config.get('password') diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index d0ba84d..bd1d2bc 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -116,6 +116,7 @@ class ExploreSchemaIn(Schema): name:str account: Dict = None prompt_id:str + source_id:str class ExplorePreviewDataIn(Schema): prompt_id:str diff --git a/core/destination_catalog.json b/core/services/destination_catalog.json similarity index 100% rename from core/destination_catalog.json rename to core/services/destination_catalog.json diff --git a/core/services/explore_service.py b/core/services/explore_service.py new file mode 100644 index 0000000..dd08075 --- /dev/null +++ b/core/services/explore_service.py @@ -0,0 +1,170 @@ +import json +import logging +import os +from typing import List, Union +import uuid +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from os.path import dirname, join +from core.api import create_new_run +from core.models import Credential, Destination, OAuthApiKeys, Source, SourceAccessInfo, StorageCredentials, Sync, Workspace +logger = logging.getLogger(__name__) + +SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] +class ExploreService: + @staticmethod + def create_spreadsheet(name:str,refresh_token:str)->str: + logger.debug("create_spreadsheet") + credentials_dict = { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": refresh_token + } + # Create a Credentials object from the dictionary + try: + base_spreadsheet_url = "https://docs.google.com/spreadsheets/d/" + credentials = Credentials.from_authorized_user_info( + credentials_dict, scopes=SPREADSHEET_SCOPES + ) + service = build("sheets", "v4", credentials=credentials) + # Create the spreadsheet + spreadsheet = {"properties": {"title": name}} + spreadsheet = ( + service.spreadsheets() + .create(body=spreadsheet, fields="spreadsheetId") + .execute() + ) + spreadsheet_id = spreadsheet.get("spreadsheetId") + #Update the sharing settings to make the spreadsheet publicly accessible + drive_service = build('drive', 'v3', credentials=credentials) + drive_service.permissions().create( + fileId=spreadsheet_id, + body={ + "role": "writer", + "type": "anyone", + "withLink": True + }, + fields="id" + ).execute() + + spreadsheet_url = f"{base_spreadsheet_url}{spreadsheet_id}" + return spreadsheet_url + except Exception as e: + logger.exception(f"Error creating spreadsheet: {e}") + raise Exception("spreadhseet creation failed") + + @staticmethod + def create_source(shopify_source_id:str,workspace_id:str,account:object)->object: + try: + #creating source credentail + credential = {"id": uuid.uuid4()} + credential["workspace"] = Workspace.objects.get(id=workspace_id) + credential["connector_id"] = "SRC_POSTGRES" + credential["name"] = "SRC_POSTGRES" + credential["account"] = account + credential["status"] = "active" + source_access_info = SourceAccessInfo.objects.get(source_id=shopify_source_id) + storage_credential = StorageCredentials.objects.get(id = source_access_info.storage_credentials.id) + connector_config = { + "ssl": False, + "host": storage_credential.connector_config["host"], + "port": storage_credential.connector_config["port"], + "user": storage_credential.connector_config["username"], + "database": storage_credential.connector_config["database"], + "password": storage_credential.connector_config["password"], + "namespace": storage_credential.connector_config["namespace"], + } + credential["connector_config"] = connector_config + cred = Credential.objects.create(**credential) + source = { + "name":"SRC_POSTGRES", + "id":uuid.uuid4() + } + #creating source object + source["workspace"] = Workspace.objects.get(id=workspace_id) + source["credential"] = Credential.objects.get(id=cred.id) + source_catalog = {} + json_file_path = join(dirname(__file__), 'source_catalog.json') + with open(json_file_path, 'r') as openfile: + source_catalog = json.load(openfile) + source["catalog"] = source_catalog + source["status"] = "active" + result = Source.objects.create(**source) + return result + except Exception as e: + logger.exception(f"Error creating source: {e}") + raise Exception("unable to create source") + + @staticmethod + def create_destination(spreadsheet_name:str,workspace_id:str,account:object)->List[Union[str, object]]: + try: + #creating destination credential + oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") + credential = {"id": uuid.uuid4()} + credential["workspace"] = Workspace.objects.get(id=workspace_id) + credential["connector_id"] = "DEST_GOOGLE-SHEETS" + credential["name"] = "DEST_GOOGLE-SHEETS" + credential["account"] = account + credential["status"] = "active" + spreadsheet_url = ExploreService.create_spreadsheet(spreadsheet_name,refresh_token=oauthkeys.oauth_config["refresh_token"]) + connector_config = { + "spreadsheet_id": spreadsheet_url, + "credentials": { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": oauthkeys.oauth_config["refresh_token"], + }, + } + credential["connector_config"] = connector_config + cred = Credential.objects.create(**credential) + destination= { + "name":"DEST_GOOGLE-SHEETS", + "id":uuid.uuid4() + + } + #creating destination object + destination["workspace"] = Workspace.objects.get(id=workspace_id) + destination["credential"] = Credential.objects.get(id=cred.id) + destination_catalog = {} + json_file_path = join(dirname(__file__), 'destination_catalog.json') + with open(json_file_path, 'r') as openfile: + destination_catalog = json.load(openfile) + destination["catalog"] = destination_catalog + result = Destination.objects.create(**destination) + logger.info(result) + return [spreadsheet_url,result] + except Exception as e: + logger.exception(f"Error creating destination: {e}") + raise Exception("unable to create destination") + + @staticmethod + def create_sync(source:object,destination:object,workspace_id:str)->object: + try: + logger.debug("creating sync in service") + logger.debug(source.id) + sync_config = { + "name":"Warehouse to sheets", + "id":uuid.uuid4(), + "status":"active", + "ui_state":{} + + } + schedule = {"run_interval": 3600000} + sync_config["schedule"] = schedule + sync_config["source"] = source + sync_config["destination"] = destination + sync_config["workspace"] = Workspace.objects.get(id=workspace_id) + sync = Sync.objects.create(**sync_config) + return sync + except Exception as e: + logger.exception(f"Error creating sync: {e}") + raise Exception("unable to create sync") + + @staticmethod + def create_run(request:object,workspace_id:str,sync_id:str,payload:object)->None: + try: + response = create_new_run(request,workspace_id,sync_id,payload) + logger.debug(response) + except Exception as e: + logger.exception(f"Error creating run: {e}") + raise Exception("unable to create run") \ No newline at end of file diff --git a/core/source_catalog.json b/core/services/source_catalog.json similarity index 100% rename from core/source_catalog.json rename to core/services/source_catalog.json From 999ad3be6d295a78c69d535f300abe4a31a8198e Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Wed, 8 May 2024 16:35:48 +0530 Subject: [PATCH 065/159] prompt template and build --- core/prompt_api.py | 2 +- .../order_with_product.liquid | 9 ------- core/services/prompt_templates/prompts.liquid | 16 +++++++++++++ core/services/prompts.py | 24 +++++++++++++++---- 4 files changed, 36 insertions(+), 15 deletions(-) delete mode 100644 core/services/prompt_templates/order_with_product.liquid create mode 100644 core/services/prompt_templates/prompts.liquid diff --git a/core/prompt_api.py b/core/prompt_api.py index 36d61a0..e39e401 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -49,7 +49,7 @@ def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) +@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): prompt = Prompt.objects.get(id=prompt_id) query = PromptService().build(prompt.table, prompt_req.time_window, prompt_req.filters) diff --git a/core/services/prompt_templates/order_with_product.liquid b/core/services/prompt_templates/order_with_product.liquid deleted file mode 100644 index 45bba88..0000000 --- a/core/services/prompt_templates/order_with_product.liquid +++ /dev/null @@ -1,9 +0,0 @@ - -{% assign filterStr = "" %} -{% for filter in filters %} - {% assign filterStr = filterStr| append: filter.name | append: " " | append: filter.operator | append: " "| append: filter.value %} - {% if forloop.last == false %} - {% assign filterStr = filterStr| append: " and " %} - {% endif %} -{% endfor %} -select * from {{ table }} where {{ filterStr }} and updated_at >= {{ timeWindow.range.start}} and updated_at <= {{ timeWindow.range.end }} \ No newline at end of file diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid new file mode 100644 index 0000000..740a35e --- /dev/null +++ b/core/services/prompt_templates/prompts.liquid @@ -0,0 +1,16 @@ + +{% assign filterStr = "" %} +{% for filter in filters %} + {% capture name -%} + "{{ filter.name }}" + {%- endcapture -%} + {% capture value -%} + '{{ filter.value }}' + {%- endcapture -%} + {% assign operator = filter.operator %} + {% assign filterStr = filterStr| append: name | append: " " | append: operator | append: " "| append: value %} + {% if forloop.last == false %} + {% assign filterStr = filterStr| append: " and " %} + {% endif %} +{% endfor %} +select * from {{ table }} where {{ filterStr }} and "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file diff --git a/core/services/prompts.py b/core/services/prompts.py index 4f56ea4..48d41b0 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,16 +1,30 @@ -from liquid import Environment, FileSystemLoader +import logging +from pathlib import Path +from liquid import Environment, FileSystemLoader, Mode, StrictUndefined +from core.schemas.prompt import TimeWindow + +logger = logging.getLogger(__name__) class PromptService(): @classmethod - def getTemplateFile(cls, tab:str): + def getTemplateFile(cls): return 'prompts.liquid' @staticmethod - def build(table, timeWindow, filters)-> str: + def build(table, timeWindow: TimeWindow , filters: list[TimeWindow])-> str: + timeWindowDict = timeWindow.dict() + filterList = [] + for filter in filters: + filterList.append(filter.__dict__) file_name = PromptService.getTemplateFile() - env = Environment(loader=FileSystemLoader("templates/")) + template_parent_path = Path(__file__).parent.absolute() + env = Environment( + tolerance=Mode.STRICT, + undefined=StrictUndefined, + loader=FileSystemLoader(str(template_parent_path) + "/prompt_templates")) template = env.get_template(file_name) - return template.render(table=table, timeWindow=timeWindow, filters=filters) + filters = list(filters) + return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) From d272e67a4cd0a7446a49be00a43a0dac3be3b2f1 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 9 May 2024 14:37:11 +0530 Subject: [PATCH 066/159] "feat: Align Preview Data Endpoint with SourceAccessTable Creation" --- core/prompt_api.py | 26 ++++++++++++------- core/services/prompt_templates/prompts.liquid | 6 ++++- core/services/prompts.py | 13 +++++----- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index e39e401..4e3b84d 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -8,7 +8,7 @@ from ninja import Router from pydantic import Json -from core.models import Credential, Prompt, SourceAccessInfo, StorageCredentials +from core.models import Credential, Prompt, Source, SourceAccessInfo, StorageCredentials from core.schemas.prompt import PromptPreviewSchemaIn from core.schemas.schemas import DetailSchema, PromptSchema, PromptSchemaOut from core.services.prompts import PromptService @@ -34,11 +34,16 @@ def get_prompts(request): logger.exception("prompts listing error:"+ err) return (400, {"detail": "The list of prompts cannot be fetched."}) -@router.get("/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) -def get_prompts(request,prompt_id): +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) +def get_prompts(request,workspace_id,prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) + credential_info = Source.objects.filter(credential__workspace_id=workspace_id, credential__connector_id=prompt.type).select_related('credential').values('credential__name', 'credential__created_at', 'id') + logger.debug(credential_info[0]) + for info in credential_info.all(): + source_id = str(info['credential__name'] +'$'+ str(info['credential__created_at'])) + prompt.spec["properties"]["sourceId"]["enum"].append(source_id) return prompt except Exception: logger.exception("prompt listing error") @@ -52,16 +57,19 @@ def custom_serializer(obj): @router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): prompt = Prompt.objects.get(id=prompt_id) - query = PromptService().build(prompt.table, prompt_req.time_window, prompt_req.filters) - souce_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) - storage_cred = StorageCredentials.objects.get(id=souce_access_info.storage_credentials_id) + source_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) + storage_credentials = StorageCredentials.objects.get(id=source_access_info.storage_credentials.id) + schema_name = storage_credentials.connector_config["schema"] + table_name = f'{schema_name}.{prompt.table}' + query = PromptService().build(table_name, prompt_req.time_window, prompt_req.filters) + logger.debug(query) host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = storage_cred.connector_config.get('password') - db_username = storage_cred.connector_config.get('username') + db_password = storage_credentials.connector_config.get('password') + db_username = storage_credentials.connector_config.get('username') conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) cursor = conn.cursor() cursor.execute(query) + items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] conn.commit() conn.close() - items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] return json.dumps(items, indent=4, default=custom_serializer) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 740a35e..22710a7 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -13,4 +13,8 @@ {% assign filterStr = filterStr| append: " and " %} {% endif %} {% endfor %} -select * from {{ table }} where {{ filterStr }} and "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file +{% if filterStr !="" %} + {% assign filterStr = filterStr | append: " and " %} +{% endif %} + +select * from {{ table }} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' limit 100 \ No newline at end of file diff --git a/core/services/prompts.py b/core/services/prompts.py index 48d41b0..5631c86 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -3,7 +3,7 @@ from liquid import Environment, FileSystemLoader, Mode, StrictUndefined -from core.schemas.prompt import TimeWindow +from core.schemas.prompt import TimeWindow, Filter logger = logging.getLogger(__name__) class PromptService(): @@ -11,17 +11,18 @@ class PromptService(): def getTemplateFile(cls): return 'prompts.liquid' @staticmethod - def build(table, timeWindow: TimeWindow , filters: list[TimeWindow])-> str: + def build(table, timeWindow: TimeWindow , filters: list[Filter]) -> str: timeWindowDict = timeWindow.dict() - filterList = [] - for filter in filters: - filterList.append(filter.__dict__) + filterList = [filter.__dict__ for filter in filters] file_name = PromptService.getTemplateFile() template_parent_path = Path(__file__).parent.absolute() env = Environment( tolerance=Mode.STRICT, undefined=StrictUndefined, - loader=FileSystemLoader(str(template_parent_path) + "/prompt_templates")) + loader=FileSystemLoader( + f"{str(template_parent_path)}/prompt_templates" + ), + ) template = env.get_template(file_name) filters = list(filters) return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) From dcbc51b45c2f0be5d081a62f8800fc1762a14d29 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 9 May 2024 16:17:41 +0530 Subject: [PATCH 067/159] feat: removed limit on records in liquid template --- core/prompt_api.py | 6 +++++- core/services/prompt_templates/prompts.liquid | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index 4e3b84d..6bb0594 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -56,8 +56,12 @@ def custom_serializer(obj): @router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): + name,created_at = prompt_req.source_id.split("$") + logger.debug(name,created_at) + credential = Credential.objects.get(name=name,created_at=created_at) + source_id = Source.objects.get(credential_id = credential.id) prompt = Prompt.objects.get(id=prompt_id) - source_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) + source_access_info = SourceAccessInfo.objects.get(source_id=source_id) storage_credentials = StorageCredentials.objects.get(id=source_access_info.storage_credentials.id) schema_name = storage_credentials.connector_config["schema"] table_name = f'{schema_name}.{prompt.table}' diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 22710a7..1380c82 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -17,4 +17,4 @@ {% assign filterStr = filterStr | append: " and " %} {% endif %} -select * from {{ table }} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' limit 100 \ No newline at end of file +select * from {{ table }} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file From b3b5511d9af535b93ab30e37c4854fe48228bef5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 10 May 2024 10:49:34 +0530 Subject: [PATCH 068/159] feat: removed source_id from prompt spec --- core/prompt_api.py | 16 +++++++++------- core/schemas/schemas.py | 3 ++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index 6bb0594..2ae34e2 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -41,9 +41,15 @@ def get_prompts(request,workspace_id,prompt_id): prompt = Prompt.objects.get(id=prompt_id) credential_info = Source.objects.filter(credential__workspace_id=workspace_id, credential__connector_id=prompt.type).select_related('credential').values('credential__name', 'credential__created_at', 'id') logger.debug(credential_info[0]) + source_ids = [] for info in credential_info.all(): - source_id = str(info['credential__name'] +'$'+ str(info['credential__created_at'])) - prompt.spec["properties"]["sourceId"]["enum"].append(source_id) + source={ + "name" : str(info['credential__name'] +'$'+ str(info['credential__created_at'])), + "id":str(info['id']) + } + source_ids.append(source) + prompt.source_id = source_ids + logger.debug(prompt) return prompt except Exception: logger.exception("prompt listing error") @@ -56,12 +62,8 @@ def custom_serializer(obj): @router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): - name,created_at = prompt_req.source_id.split("$") - logger.debug(name,created_at) - credential = Credential.objects.get(name=name,created_at=created_at) - source_id = Source.objects.get(credential_id = credential.id) prompt = Prompt.objects.get(id=prompt_id) - source_access_info = SourceAccessInfo.objects.get(source_id=source_id) + source_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) storage_credentials = StorageCredentials.objects.get(id=source_access_info.storage_credentials.id) schema_name = storage_credentials.connector_config["schema"] table_name = f'{schema_name}.{prompt.table}' diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index bd1d2bc..199c41f 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -7,7 +7,7 @@ """ from datetime import datetime -from typing import Dict, Optional +from typing import Dict, List, Optional from django.contrib.auth import get_user_model from ninja import Field, ModelSchema, Schema @@ -72,6 +72,7 @@ class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt model_fields = ["id","name","description","type","spec","package_id","gated","table"] + source_id: List[Dict[str, str]] class PromptSchemaOut(Schema): id: str From e32aa2eeb7d6466ad5e8c94612a98c7b54e2da46 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 10 May 2024 11:00:04 +0530 Subject: [PATCH 069/159] feat: renamed source_ids to source in promptSchema --- core/prompt_api.py | 6 +++--- core/schemas/schemas.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index 2ae34e2..b361bb6 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -41,14 +41,14 @@ def get_prompts(request,workspace_id,prompt_id): prompt = Prompt.objects.get(id=prompt_id) credential_info = Source.objects.filter(credential__workspace_id=workspace_id, credential__connector_id=prompt.type).select_related('credential').values('credential__name', 'credential__created_at', 'id') logger.debug(credential_info[0]) - source_ids = [] + sources = [] for info in credential_info.all(): source={ "name" : str(info['credential__name'] +'$'+ str(info['credential__created_at'])), "id":str(info['id']) } - source_ids.append(source) - prompt.source_id = source_ids + sources.append(source) + prompt.sources = sources logger.debug(prompt) return prompt except Exception: diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 199c41f..cfbd715 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -72,7 +72,7 @@ class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt model_fields = ["id","name","description","type","spec","package_id","gated","table"] - source_id: List[Dict[str, str]] + sources: List[Dict[str, str]] class PromptSchemaOut(Schema): id: str From 2f77c9f213d9f0cc93297f5b5acb66da3991ef39 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 10 May 2024 11:10:39 +0530 Subject: [PATCH 070/159] feat: renamed shopify_store to name in ConnectionSchema --- core/api.py | 2 +- core/schemas/schemas.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/api.py b/core/api.py index 5c392d2..7bf70a3 100644 --- a/core/api.py +++ b/core/api.py @@ -206,7 +206,7 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): def create_connection_with_default_warehouse(request, workspace_id,payload: ConnectionSchemaIn): data = payload.dict() try: - source_credential_payload = CredentialSchemaIn(name=data["shopify_store"],account=data["account"],connector_type=data["source_connector_type"],connector_config=data["source_connector_config"]) + source_credential_payload = CredentialSchemaIn(name=data["name"],account=data["account"],connector_type=data["source_connector_type"],connector_config=data["source_connector_config"]) source_credential = create_credential(request,workspace_id,source_credential_payload) source_payload = SourceSchemaIn(name="shopify",credential_id=source_credential.id,catalog = data["source_catalog"]) source = create_source(request,workspace_id,source_payload) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index cfbd715..4ed24b8 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -103,7 +103,7 @@ class CredentialSchemaIn(Schema): class ConnectionSchemaIn(Schema): account: Dict = None - shopify_store: str + name: str source_catalog: Dict destination_catalog: Dict schedule: Dict From 6f793578076ecf16ef217cd7d7ec921416e1b2aa Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 14 May 2024 13:19:30 +0530 Subject: [PATCH 071/159] feat: Added filters for prompts --- core/prompt_api.py | 54 ++++++++++++++++++++++++++-------------- core/schemas/prompt.py | 2 +- core/schemas/schemas.py | 9 +++++-- core/services/prompts.py | 7 +++++- init_db/prompt_init.py | 4 ++- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index b361bb6..ca57dfc 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -3,14 +3,12 @@ import logging import os from typing import List - import psycopg2 from ninja import Router from pydantic import Json - -from core.models import Credential, Prompt, Source, SourceAccessInfo, StorageCredentials +from core.models import Credential, Prompt, Source, StorageCredentials from core.schemas.prompt import PromptPreviewSchemaIn -from core.schemas.schemas import DetailSchema, PromptSchema, PromptSchemaOut +from core.schemas.schemas import DetailSchema, PromptByIdSchema, PromptSchemaOut from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -34,21 +32,37 @@ def get_prompts(request): logger.exception("prompts listing error:"+ err) return (400, {"detail": "The list of prompts cannot be fetched."}) -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) -def get_prompts(request,workspace_id,prompt_id): +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptByIdSchema, 400: DetailSchema}) +def get_prompt(request,workspace_id,prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) - credential_info = Source.objects.filter(credential__workspace_id=workspace_id, credential__connector_id=prompt.type).select_related('credential').values('credential__name', 'credential__created_at', 'id') - logger.debug(credential_info[0]) - sources = [] - for info in credential_info.all(): - source={ - "name" : str(info['credential__name'] +'$'+ str(info['credential__created_at'])), - "id":str(info['id']) - } - sources.append(source) - prompt.sources = sources + if not PromptService.is_prompt_enabled(workspace_id,prompt): + detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" + return 400, {"detail": detail_message} + credential_info = Source.objects.filter( + credential__workspace_id=workspace_id, + credential__connector_id=prompt.type + ).select_related('credential', 'source_access_info') + schemas = {} + + for info in credential_info: + if source_access_info := info.source_access_info.first(): + storage_id = source_access_info.storage_credentials.id + if storage_id not in schemas: + schema = { + "id": str(storage_id), + "name": source_access_info.storage_credentials.connector_config["schema"], + "sources": [], + } + schemas[storage_id] = schema + schemas[storage_id]["sources"].append({ + "name": f"{info.credential.name}${info.credential.created_at}", + "id": str(info.id), + }) + # Convert schemas dictionary to a list (optional) + final_schemas = list(schemas.values()) + prompt.schemas = final_schemas logger.debug(prompt) return prompt except Exception: @@ -60,11 +74,13 @@ def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() -@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 404: DetailSchema}) +@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 400: DetailSchema}) def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): prompt = Prompt.objects.get(id=prompt_id) - source_access_info = SourceAccessInfo.objects.get(source_id=prompt_req.source_id) - storage_credentials = StorageCredentials.objects.get(id=source_access_info.storage_credentials.id) + if not PromptService.is_prompt_enabled(workspace_id,prompt): + detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" + return 400, {"detail": detail_message} + storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] table_name = f'{schema_name}.{prompt.table}' query = PromptService().build(table_name, prompt_req.time_window, prompt_req.filters) diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index db280a1..2503aa9 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -18,7 +18,7 @@ class Filter(Schema): class PromptPreviewSchemaIn(Schema): - source_id: str + schema_id: str time_window: TimeWindow filters: list[Filter] diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 4ed24b8..a0d7a17 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -72,7 +72,12 @@ class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt model_fields = ["id","name","description","type","spec","package_id","gated","table"] - sources: List[Dict[str, str]] + +class PromptByIdSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Prompt + model_fields = ["id","name","description","type","spec","package_id","gated","table"] + schemas: List[Dict] class PromptSchemaOut(Schema): id: str @@ -117,7 +122,7 @@ class ExploreSchemaIn(Schema): name:str account: Dict = None prompt_id:str - source_id:str + schema_id:str class ExplorePreviewDataIn(Schema): prompt_id:str diff --git a/core/services/prompts.py b/core/services/prompts.py index 5631c86..4f7525d 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,6 +1,7 @@ import logging from pathlib import Path +from core.models import Credential from liquid import Environment, FileSystemLoader, Mode, StrictUndefined from core.schemas.prompt import TimeWindow, Filter @@ -27,7 +28,11 @@ def build(table, timeWindow: TimeWindow , filters: list[Filter]) -> str: filters = list(filters) return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) - + @staticmethod + def is_prompt_enabled(workspace_id:str,prompt:object) -> bool: + connector_ids = list(Credential.objects.filter(workspace_id=workspace_id).values('connector_id').distinct()) + connector_types = [connector['connector_id'] for connector in connector_ids] + return prompt.type in connector_types \ No newline at end of file diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index ebb128a..4d4276d 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -12,11 +12,13 @@ import requests import os from requests.auth import HTTPBasicAuth - +import logging +logger = logging.getLogger(__name__) prompt_defs = json.loads(open(join(dirname(__file__), "test_prompt_def.json"), "r").read()) for prompt_def in prompt_defs["definitions"]: + logger.debug(prompt_def) resp = requests.post( f"http://localhost:{os.environ['PORT']}/api/v1/superuser/prompts/create", json={ From 8a0cbae6dd236dcd60ff7de35c2fe8e5f19ef999 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Thu, 16 May 2024 10:19:49 +0530 Subject: [PATCH 072/159] feat: add an optional user meta details field for SocialUser --- core/schemas.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/schemas.py b/core/schemas.py index 387e43e..07b369b 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -8,16 +8,13 @@ from datetime import datetime from typing import Dict, Optional - -from django.contrib.auth import get_user_model +from enum import Enum +from django.contrib.auth.models import User from ninja import Field, ModelSchema, Schema from pydantic import UUID4 from .models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys -User = get_user_model() - - def camel_to_snake(s): return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_") @@ -268,9 +265,22 @@ class SocialAccount(Schema): scope: str token_type: str id_token: str + + +class UserRole(Enum): + ENGINEERING = 'engineering' + MARKETING = 'marketing' + OTHER = 'other' + +class UserMetaDetails(Schema): + role: UserRole + promotions: bool + class SocialUser(Schema): name: str email: str + meta: Optional[UserMetaDetails] + class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser From 69838266295e51e6215dda5377c0a4c79cf97d64 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 16 May 2024 16:15:31 +0530 Subject: [PATCH 073/159] feat: generate source PostgreSQL catalog via discover call and create explores using schema ID --- core/api.py | 48 +++++---- core/explore_api.py | 50 ++++----- core/schemas/schemas.py | 41 ++++--- core/services/explore_service.py | 100 ++++++++++------- core/services/source_catalog.json | 172 ------------------------------ init_db/test_prompt_def.json | 86 +++++++-------- 6 files changed, 186 insertions(+), 311 deletions(-) diff --git a/core/api.py b/core/api.py index 7bf70a3..f3403df 100644 --- a/core/api.py +++ b/core/api.py @@ -8,6 +8,7 @@ import json import logging +import time import uuid import uuid from datetime import datetime @@ -200,42 +201,47 @@ def create_credential(request, workspace_id, payload: CredentialSchemaIn): except Exception: logger.exception("Credential error") return {"detail": "The specific credential cannot be created."} - + @router.post("/workspaces/{workspace_id}/connection/DefaultWarehouse", response={200: SuccessSchema, 400: DetailSchema}) -def create_connection_with_default_warehouse(request, workspace_id,payload: ConnectionSchemaIn): +def create_connection_with_default_warehouse(request, workspace_id, payload: ConnectionSchemaIn): data = payload.dict() try: - source_credential_payload = CredentialSchemaIn(name=data["name"],account=data["account"],connector_type=data["source_connector_type"],connector_config=data["source_connector_config"]) - source_credential = create_credential(request,workspace_id,source_credential_payload) - source_payload = SourceSchemaIn(name="shopify",credential_id=source_credential.id,catalog = data["source_catalog"]) - source = create_source(request,workspace_id,source_payload) - workspace = Workspace.objects.get(id = workspace_id) + source_credential_payload = CredentialSchemaIn( + name=data["name"], account=data["account"], connector_type=data["source_connector_type"], connector_config=data["source_connector_config"]) + source_credential = create_credential(request, workspace_id, source_credential_payload) + source_payload = SourceSchemaIn(name="shopify", credential_id=source_credential.id, + catalog=data["source_catalog"]) + source = create_source(request, workspace_id, source_payload) + workspace = Workspace.objects.get(id=workspace_id) storage_credentials = DefaultWarehouse.create(workspace) - source_access_info = {"source":source, "storage_credentials":storage_credentials} - SourceAccessInfo.objects.create(**source_access_info) - destination_credential_payload = CredentialSchemaIn(name="default warehouse",account=data["account"],connector_type="DEST_POSTGRES-DEST",connector_config=storage_credentials.connector_config) - destination_credential = create_credential(request,workspace_id,destination_credential_payload) - destination_payload = DestinationSchemaIn(name="default warehouse",credential_id=destination_credential.id,catalog = data["destination_catalog"]) - destination = create_destination(request,workspace_id,destination_payload) - sync_payload = SyncSchemaIn(name="shopify to default warehouse",source_id=source.id,destination_id=destination.id,schedule=data["schedule"]) - sync = create_sync(request,workspace_id,sync_payload) + source_access_info = {"source": source, "storage_credentials": storage_credentials} + SourceAccessInfo.objects.create(**source_access_info) + destination_credential_payload = CredentialSchemaIn( + name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + destination_credential = create_credential(request, workspace_id, destination_credential_payload) + destination_payload = DestinationSchemaIn( + name="default warehouse", credential_id=destination_credential.id, catalog=data["destination_catalog"]) + destination = create_destination(request, workspace_id, destination_payload) + sync_payload = SyncSchemaIn(name="shopify to default warehouse", source_id=source.id, + destination_id=destination.id, schedule=data["schedule"]) + sync = create_sync(request, workspace_id, sync_payload) run_payload = SyncStartStopSchemaIn(full_refresh=True) - response = create_new_run(request,workspace_id,sync.id,run_payload) + time.sleep(6) + response = create_new_run(request, workspace_id, sync.id, run_payload) logger.debug(response) return "starting sync from shopify to default warehouse" except Exception as e: logger.exception(e) return {"detail": "The specific connection cannot be created."} -@router.get("/workspaces/{workspace_id}/storage-credentials",response={200: Json, 400: DetailSchema}) + +@router.get("/workspaces/{workspace_id}/storage-credentials", response={200: Json, 400: DetailSchema}) def storage_credentials(request, workspace_id): - config={} + config = {} logger.info("came here in storeage") try: - creds = StorageCredentials.objects.filter(workspace_id=workspace_id).get( - connector_config__shopify_store="chitumalla-store" - ) + creds = StorageCredentials.objects.get(workspace_id=workspace_id) config['username'] = creds.connector_config["username"] config['password'] = creds.connector_config["password"] config["namespace"] = creds.connector_config["namespace"] diff --git a/core/explore_api.py b/core/explore_api.py index 249e6c1..84ccf0c 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,15 +1,13 @@ import json import logging import json +import time from typing import List import uuid from decouple import config -import time import json from pydantic import Json import requests - - from core.models import Account, Explore, Prompt, Workspace from core.schemas.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn from ninja import Router @@ -19,19 +17,20 @@ router = Router() ACTIVATION_URL = config("ACTIVATION_SERVER") + @router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) -def get_explores(request,workspace_id): +def get_explores(request, workspace_id): try: logger.debug("listing explores") workspace = Workspace.objects.get(id=workspace_id) - return Explore.objects.filter(workspace=workspace) + return Explore.objects.filter(workspace=workspace).order_by('created_at') except Exception: logger.exception("explores listing error") return (400, {"detail": "The list of explores cannot be fetched."}) -@router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) -def create_explore(request, workspace_id,payload: ExploreSchemaIn): +@router.post("/workspaces/{workspace_id}/create", response={200: ExploreSchema, 400: DetailSchema}) +def create_explore(request, workspace_id, payload: ExploreSchemaIn): data = payload.dict() try: data["id"] = uuid.uuid4() @@ -45,35 +44,36 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): account_info["workspace"] = data["workspace"] account = Account.objects.create(**account_info) data["account"] = account - #create source - logger.debug("source_id is %s",data["source_id"]) - source = ExploreService.create_source(data["source_id"],workspace_id,account) - #create destination + # create source + source = ExploreService.create_source(data["schema_id"], data["query"], workspace_id, account) + time.sleep(6) + # create destination spreadsheet_name = f"valmiio {prompt.name} sheet" - destination_data = ExploreService.create_destination(spreadsheet_name,workspace_id,account) + destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) spreadsheet_url = destination_data[0] destination = destination_data[1] - logger.debug("after service creation") - logger.info(destination.id) - #create sync - sync = ExploreService.create_sync(source,destination,workspace_id) + # create sync + sync = ExploreService.create_sync(source, destination, workspace_id) + # await asyncio.sleep(5) time.sleep(5) - #creating explore - data.pop("source_id") + # creating explore + del data["schema_id"] + del data["query"] data["name"] = f"valmiio {prompt.name}" data["sync"] = sync data["spreadsheet_url"] = spreadsheet_url - explore = Explore.objects.create(**data) - #create run + explore = Explore.objects.create(**data) + # create run payload = SyncStartStopSchemaIn(full_refresh=True) - ExploreService.create_run(request,workspace_id,sync.id,payload) + ExploreService.create_run(request, workspace_id, sync.id, payload) return explore except Exception as e: logger.exception(e) return (400, {"detail": "The specific explore cannot be created."}) - + + @router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_explores(request,workspace_id,explore_id): +def get_explore(request, workspace_id, explore_id): try: logger.debug("listing explores") return Explore.objects.get(id=explore_id) @@ -83,7 +83,7 @@ def get_explores(request,workspace_id,explore_id): @router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) -def get_explore_status(request,workspace_id,explore_id): +def get_explore_status(request, workspace_id, explore_id): try: logger.debug("getting_explore_status") explore = Explore.objects.get(id=explore_id) @@ -95,7 +95,7 @@ def get_explore_status(request,workspace_id,explore_id): print(status) # if status == 'stopped': # CODE for re running the sync from backend - # payload = SyncStartStopSchemaIn(full_refresh=True) + # payload = SyncStartStopSchemaIn(full_refresh=True) # response = create_new_run(request,workspace_id,sync_id,payload) # print(response) # return "sync got failed. Please re-try again" diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index a0d7a17..e5a90b8 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -44,7 +44,7 @@ class Config(CamelSchemaConfig): class UserSchemaOut(ModelSchema): class Config(CamelSchemaConfig): model = User - model_fields = ["first_name", "email","username"] + model_fields = ["first_name", "email", "username"] organizations: list[OrganizationSchema] = None @@ -52,10 +52,10 @@ class Config(CamelSchemaConfig): class CreatedUserSchema(ModelSchema): class Config(CamelSchemaConfig): model = User - model_fields = ["first_name", "email","username"] + model_fields = ["first_name", "email", "username"] organizations: list[OrganizationSchema] = None auth_token: str - + class ConnectorSchema(ModelSchema): class Config(CamelSchemaConfig): @@ -66,19 +66,22 @@ class Config(CamelSchemaConfig): class PackageSchema(ModelSchema): class Config(CamelSchemaConfig): model = Package - model_fields = ["name","gated","scopes"] + model_fields = ["name", "gated", "scopes"] + class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","description","type","spec","package_id","gated","table"] + model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "table"] + class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id","name","description","type","spec","package_id","gated","table"] + model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "table"] schemas: List[Dict] + class PromptSchemaOut(Schema): id: str name: str @@ -86,8 +89,11 @@ class PromptSchemaOut(Schema): type: str enabled: bool + class ExploreStatusIn(Schema): - sync_id:str + sync_id: str + + class ConnectorConfigSchemaIn(Schema): config: Dict @@ -106,6 +112,7 @@ class CredentialSchemaIn(Schema): account: Dict = None name: str + class ConnectionSchemaIn(Schema): account: Dict = None name: str @@ -114,18 +121,19 @@ class ConnectionSchemaIn(Schema): schedule: Dict source_connector_type: str source_connector_config: Dict - - + class ExploreSchemaIn(Schema): ready: bool = False - name:str + name: str account: Dict = None - prompt_id:str - schema_id:str + prompt_id: str + schema_id: str + query: str + class ExplorePreviewDataIn(Schema): - prompt_id:str + prompt_id: str class CredentialSchemaUpdateIn(Schema): @@ -166,7 +174,6 @@ class Config(CamelSchemaConfig): workspace: WorkspaceSchema = Field(None, alias="workspace") - class BaseSchemaIn(Schema): workspace_id: UUID4 @@ -274,7 +281,7 @@ class Config(CamelSchemaConfig): class SocialAccount(Schema): - provider: str + provider: str type: str access_token: str expires_at: int @@ -282,9 +289,13 @@ class SocialAccount(Schema): scope: str token_type: str id_token: str + + class SocialUser(Schema): name: str email: str + + class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser diff --git a/core/services/explore_service.py b/core/services/explore_service.py index dd08075..a507994 100644 --- a/core/services/explore_service.py +++ b/core/services/explore_service.py @@ -6,14 +6,18 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from os.path import dirname, join +from decouple import config +import requests from core.api import create_new_run -from core.models import Credential, Destination, OAuthApiKeys, Source, SourceAccessInfo, StorageCredentials, Sync, Workspace +from core.models import Credential, Destination, OAuthApiKeys, Source, StorageCredentials, Sync, Workspace logger = logging.getLogger(__name__) - +ACTIVATION_URL = config("ACTIVATION_SERVER") SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] + + class ExploreService: @staticmethod - def create_spreadsheet(name:str,refresh_token:str)->str: + def create_spreadsheet(name: str, refresh_token: str) -> str: logger.debug("create_spreadsheet") credentials_dict = { "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], @@ -24,7 +28,7 @@ def create_spreadsheet(name:str,refresh_token:str)->str: try: base_spreadsheet_url = "https://docs.google.com/spreadsheets/d/" credentials = Credentials.from_authorized_user_info( - credentials_dict, scopes=SPREADSHEET_SCOPES + credentials_dict, scopes=SPREADSHEET_SCOPES ) service = build("sheets", "v4", credentials=credentials) # Create the spreadsheet @@ -35,7 +39,7 @@ def create_spreadsheet(name:str,refresh_token:str)->str: .execute() ) spreadsheet_id = spreadsheet.get("spreadsheetId") - #Update the sharing settings to make the spreadsheet publicly accessible + # Update the sharing settings to make the spreadsheet publicly accessible drive_service = build('drive', 'v3', credentials=credentials) drive_service.permissions().create( fileId=spreadsheet_id, @@ -43,7 +47,7 @@ def create_spreadsheet(name:str,refresh_token:str)->str: "role": "writer", "type": "anyone", "withLink": True - }, + }, fields="id" ).execute() @@ -52,19 +56,18 @@ def create_spreadsheet(name:str,refresh_token:str)->str: except Exception as e: logger.exception(f"Error creating spreadsheet: {e}") raise Exception("spreadhseet creation failed") - + @staticmethod - def create_source(shopify_source_id:str,workspace_id:str,account:object)->object: + def create_source(schema_id: str, query: str, workspace_id: str, account: object) -> object: try: - #creating source credentail + # creating source credentail credential = {"id": uuid.uuid4()} credential["workspace"] = Workspace.objects.get(id=workspace_id) credential["connector_id"] = "SRC_POSTGRES" credential["name"] = "SRC_POSTGRES" credential["account"] = account credential["status"] = "active" - source_access_info = SourceAccessInfo.objects.get(source_id=shopify_source_id) - storage_credential = StorageCredentials.objects.get(id = source_access_info.storage_credentials.id) + storage_credential = StorageCredentials.objects.get(id=schema_id) connector_config = { "ssl": False, "host": storage_credential.connector_config["host"], @@ -77,52 +80,77 @@ def create_source(shopify_source_id:str,workspace_id:str,account:object)->object credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) source = { - "name":"SRC_POSTGRES", - "id":uuid.uuid4() + "name": "SRC_POSTGRES", + "id": uuid.uuid4() } - #creating source object + # creating source object source["workspace"] = Workspace.objects.get(id=workspace_id) source["credential"] = Credential.objects.get(id=cred.id) - source_catalog = {} + url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" + body = { + "ssl": False, + "host": storage_credential.connector_config["host"], + "port": storage_credential.connector_config["port"], + "user": storage_credential.connector_config["username"], + "database": storage_credential.connector_config["database"], + "password": storage_credential.connector_config["password"], + "namespace": storage_credential.connector_config["namespace"], + "query": query + } + config = { + 'docker_image': 'valmiio/source-postgres', + 'docker_tag': 'latest', + 'config': body + } + response = requests.post(url, json=config) + response_json = response.json() json_file_path = join(dirname(__file__), 'source_catalog.json') with open(json_file_path, 'r') as openfile: source_catalog = json.load(openfile) + # TODO : HARDCODED TABLE NAME + source_catalog["streams"][0]["stream"] = response_json["catalog"]["streams"][0] + namespace = storage_credential.connector_config["namespace"] + database = storage_credential.connector_config["database"] + table = "orders_with_product_data" + source_catalog["streams"][0]["stream"]["name"] = f"{database}.{namespace}.{table}" source["catalog"] = source_catalog source["status"] = "active" + logger.debug(source_catalog) result = Source.objects.create(**source) return result except Exception as e: logger.exception(f"Error creating source: {e}") raise Exception("unable to create source") - + @staticmethod - def create_destination(spreadsheet_name:str,workspace_id:str,account:object)->List[Union[str, object]]: + def create_destination(spreadsheet_name: str, workspace_id: str, account: object) -> List[Union[str, object]]: try: - #creating destination credential - oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id,type="GOOGLE_LOGIN") + # creating destination credential + oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id, type="GOOGLE_LOGIN") credential = {"id": uuid.uuid4()} credential["workspace"] = Workspace.objects.get(id=workspace_id) credential["connector_id"] = "DEST_GOOGLE-SHEETS" credential["name"] = "DEST_GOOGLE-SHEETS" credential["account"] = account credential["status"] = "active" - spreadsheet_url = ExploreService.create_spreadsheet(spreadsheet_name,refresh_token=oauthkeys.oauth_config["refresh_token"]) + spreadsheet_url = ExploreService.create_spreadsheet( + spreadsheet_name, refresh_token=oauthkeys.oauth_config["refresh_token"]) connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { - "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], "refresh_token": oauthkeys.oauth_config["refresh_token"], }, } credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) - destination= { - "name":"DEST_GOOGLE-SHEETS", - "id":uuid.uuid4() + destination = { + "name": "DEST_GOOGLE-SHEETS", + "id": uuid.uuid4() } - #creating destination object + # creating destination object destination["workspace"] = Workspace.objects.get(id=workspace_id) destination["credential"] = Credential.objects.get(id=cred.id) destination_catalog = {} @@ -132,21 +160,21 @@ def create_destination(spreadsheet_name:str,workspace_id:str,account:object)->Li destination["catalog"] = destination_catalog result = Destination.objects.create(**destination) logger.info(result) - return [spreadsheet_url,result] + return [spreadsheet_url, result] except Exception as e: logger.exception(f"Error creating destination: {e}") raise Exception("unable to create destination") - + @staticmethod - def create_sync(source:object,destination:object,workspace_id:str)->object: + def create_sync(source: object, destination: object, workspace_id: str) -> object: try: logger.debug("creating sync in service") logger.debug(source.id) sync_config = { - "name":"Warehouse to sheets", - "id":uuid.uuid4(), - "status":"active", - "ui_state":{} + "name": "Warehouse to sheets", + "id": uuid.uuid4(), + "status": "active", + "ui_state": {} } schedule = {"run_interval": 3600000} @@ -161,10 +189,10 @@ def create_sync(source:object,destination:object,workspace_id:str)->object: raise Exception("unable to create sync") @staticmethod - def create_run(request:object,workspace_id:str,sync_id:str,payload:object)->None: + def create_run(request: object, workspace_id: str, sync_id: str, payload: object) -> None: try: - response = create_new_run(request,workspace_id,sync_id,payload) - logger.debug(response) + response = create_new_run(request, workspace_id, sync_id, payload) + logger.debug(response) except Exception as e: logger.exception(f"Error creating run: {e}") - raise Exception("unable to create run") \ No newline at end of file + raise Exception("unable to create run") diff --git a/core/services/source_catalog.json b/core/services/source_catalog.json index 0cb4e8b..3df6484 100644 --- a/core/services/source_catalog.json +++ b/core/services/source_catalog.json @@ -3,178 +3,6 @@ { "id_key": "id", "stream": { - "name": "dvdrental.dbt_transformer_p0.orders_with_product_data", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "image": {"type": "jsonb"}, - "product_type": {"type": "character varying"}, - "body_html": {"type": "character varying"}, - "title": {"type": "character varying"}, - "images": {"type": "jsonb"}, - "vendor": {"type": "character varying"}, - "options": {"type": "jsonb"}, - "product_id": {"type": "bigint"}, - "id": {"type": "bigint"}, - "name": {"type": "character varying"}, - "note": {"type": "character varying"}, - "tags": {"type": "character varying"}, - "test": {"type": "boolean"}, - "email": {"type": "character varying"}, - "phone": {"type": "character varying"}, - "token": {"type": "character varying"}, - "app_id": {"type": "bigint"}, - "number": {"type": "bigint"}, - "company": {"type": "character varying"}, - "refunds": {"type": "jsonb"}, - "user_id": {"type": "numeric"}, - "currency": {"type": "character varying"}, - "customer": {"type": "jsonb"}, - "shop_url": {"type": "character varying"}, - "closed_at": { - "type": "timestamp with time zone" - }, - "confirmed": {"type": "boolean"}, - "device_id": {"type": "character varying"}, - "po_number": {"type": "character varying"}, - "reference": {"type": "character varying"}, - "tax_lines": {"type": "jsonb"}, - "total_tax": {"type": "numeric"}, - "browser_ip": {"type": "character varying"}, - "cart_token": {"type": "character varying"}, - "created_at": { - "type": "timestamp with time zone" - }, - "deleted_at": { - "type": "timestamp with time zone" - }, - "line_items": {"type": "jsonb"}, - "source_url": {"type": "character varying"}, - "tax_exempt": {"type": "boolean"}, - "updated_at": { - "type": "timestamp with time zone" - }, - "checkout_id": {"type": "bigint"}, - "location_id": {"type": "bigint"}, - "source_name": {"type": "character varying"}, - "total_price": {"type": "numeric"}, - "cancelled_at": { - "type": "timestamp with time zone" - }, - "fulfillments": {"type": "jsonb"}, - "landing_site": {"type": "character varying"}, - "order_number": {"type": "bigint"}, - "processed_at": {"type": "character varying"}, - "total_weight": {"type": "bigint"}, - "cancel_reason": {"type": "character varying"}, - "contact_email": {"type": "character varying"}, - "payment_terms": {"type": "character varying"}, - "total_tax_set": {"type": "jsonb"}, - "checkout_token": { - "type": "character varying" - }, - "client_details": {"type": "jsonb"}, - "discount_codes": {"type": "jsonb"}, - "referring_site": { - "type": "character varying" - }, - "shipping_lines": {"type": "jsonb"}, - "subtotal_price": {"type": "numeric"}, - "taxes_included": {"type": "boolean"}, - "billing_address": {"type": "jsonb"}, - "customer_locale": { - "type": "character varying" - }, - "deleted_message": { - "type": "character varying" - }, - "estimated_taxes": {"type": "boolean"}, - "note_attributes": {"type": "jsonb"}, - "total_discounts": {"type": "numeric"}, - "total_price_set": {"type": "jsonb"}, - "total_price_usd": {"type": "numeric"}, - "financial_status": { - "type": "character varying" - }, - "landing_site_ref": { - "type": "character varying" - }, - "order_status_url": { - "type": "character varying" - }, - "shipping_address": {"type": "jsonb"}, - "current_total_tax": {"type": "numeric"}, - "source_identifier": { - "type": "character varying" - }, - "total_outstanding": {"type": "numeric"}, - "fulfillment_status": { - "type": "character varying" - }, - "subtotal_price_set": {"type": "jsonb"}, - "total_tip_received": {"type": "numeric"}, - "confirmation_number": { - "type": "character varying" - }, - "current_total_price": {"type": "numeric"}, - "deleted_description": { - "type": "character varying" - }, - "total_discounts_set": {"type": "jsonb"}, - "admin_graphql_api_id": { - "type": "character varying" - }, - "discount_allocations": {"type": "jsonb"}, - "presentment_currency": { - "type": "character varying" - }, - "current_total_tax_set": {"type": "jsonb"}, - "discount_applications": {"type": "jsonb"}, - "payment_gateway_names": {"type": "jsonb"}, - "current_subtotal_price": {"type": "numeric"}, - "total_line_items_price": {"type": "numeric"}, - "buyer_accepts_marketing": {"type": "boolean"}, - "current_total_discounts": {"type": "numeric"}, - "current_total_price_set": {"type": "jsonb"}, - "current_total_duties_set": { - "type": "character varying" - }, - "total_shipping_price_set": {"type": "jsonb"}, - "merchant_of_record_app_id": { - "type": "character varying" - }, - "original_total_duties_set": { - "type": "character varying" - }, - "current_subtotal_price_set": { - "type": "jsonb" - }, - "total_line_items_price_set": { - "type": "jsonb" - }, - "current_total_discounts_set": { - "type": "jsonb" - }, - "current_total_additional_fees_set": { - "type": "jsonb" - }, - "original_total_additional_fees_set": { - "type": "jsonb" - }, - "_airbyte_raw_id": { - "type": "character varying" - }, - "_airbyte_extracted_at": { - "type": "timestamp with time zone" - }, - "_airbyte_meta": {"type": "jsonb"} - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] }, "sync_mode": "full_refresh", "destination_sync_mode": "append" diff --git a/init_db/test_prompt_def.json b/init_db/test_prompt_def.json index b1ea308..d2f1ccc 100644 --- a/init_db/test_prompt_def.json +++ b/init_db/test_prompt_def.json @@ -1,48 +1,50 @@ { "definitions": [ - { - "name": "Inventory snapshot", - "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "table": "orders_with_product_data", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows", "sources"], - "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] - }, - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } + { + "name": "Inventory snapshot", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "table": "orders_with_product_data", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": {"type": "string"}, + "range": { + "type": "object", + "properties": { + "start": {"type": "string"}, + "end": {"type": "string"} + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": {"type": "string"}, + "name": {"type": "string"}, + "type": {"type": "string"}, + "value": {"type": "string"}, + "operator": {"type": "string", "enum": ["=", "!="]} + }, + "required": ["label", "name", "type", "value", "operator"] + } + } } - } }, - "filters": { - "type": "array", - "anyOf": [ - { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - } - ] - } - } - }, - "type": "SRC_SHOPIFY", - "package_id": "P0", - "gated": true - } + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + } ] } From 3b764e9e7fcc3c0353b87fde2ffd31458dc35810 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 16 May 2024 16:46:39 +0530 Subject: [PATCH 074/159] feat: generate source PostgreSQL catalog via discover call and create explore using schema id --- core/explore_api.py | 1 + core/schemas/schemas.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/core/explore_api.py b/core/explore_api.py index 84ccf0c..dc236c8 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -61,6 +61,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): del data["query"] data["name"] = f"valmiio {prompt.name}" data["sync"] = sync + data["ready"] = False data["spreadsheet_url"] = spreadsheet_url explore = Explore.objects.create(**data) # create run diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index e5a90b8..d154212 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -124,7 +124,6 @@ class ConnectionSchemaIn(Schema): class ExploreSchemaIn(Schema): - ready: bool = False name: str account: Dict = None prompt_id: str From 8a3a0ac42f14ff2554f4da8a59e46525f1d9cd7f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 16 May 2024 17:16:05 +0530 Subject: [PATCH 075/159] feat: generate source PostgreSQL catalog via discover call and create explore using schema id --- core/explore_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index dc236c8..9b21def 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -88,8 +88,10 @@ def get_explore_status(request, workspace_id, explore_id): try: logger.debug("getting_explore_status") explore = Explore.objects.get(id=explore_id) + response = {} if explore.ready: - return "sync completed" + response["status"] = "success" + return json.dumps(response) sync_id = explore.sync.id response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latestRunStatus") status = response.text @@ -100,7 +102,6 @@ def get_explore_status(request, workspace_id, explore_id): # response = create_new_run(request,workspace_id,sync_id,payload) # print(response) # return "sync got failed. Please re-try again" - response = {} if status == '"running"': response["status"] = "running" return json.dumps(response) From b2b2cddd019f17f54d422baf80336504280e7d39 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 16 May 2024 17:20:06 +0530 Subject: [PATCH 076/159] feat: generate source PostgreSQL catalog via discover call and create explore using schema id --- core/explore_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/explore_api.py b/core/explore_api.py index 9b21def..5b57e2d 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -103,9 +103,11 @@ def get_explore_status(request, workspace_id, explore_id): # print(response) # return "sync got failed. Please re-try again" if status == '"running"': + explore.ready = False response["status"] = "running" return json.dumps(response) if status == '"failed"': + explore.ready = False response["status"] = "failed" return json.dumps(response) explore.ready = True From fcffd2683139357782f0d2fd5d53643cbd6772f8 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Thu, 16 May 2024 18:43:10 +0530 Subject: [PATCH 077/159] fix: update models.py --- core/models.py | 3 +-- core/schemas.py | 12 +----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/core/models.py b/core/models.py index 19ef82a..3c5eec5 100644 --- a/core/models.py +++ b/core/models.py @@ -9,7 +9,6 @@ from django.utils import timezone import uuid -from django.contrib.auth import get_user_model from django.contrib.auth.models import AbstractUser from django.db import models from django.contrib.postgres.fields import ArrayField @@ -34,6 +33,7 @@ class User(AbstractUser): USERNAME_FIELD = "email" REQUIRED_FIELDS = [] organizations = models.ManyToManyField(to="Organization", related_name="users", blank=True) + meta = models.JSONField(default=dict) def __str__(self): return f"{self.email} - {self.first_name} {self.last_name} - {self.organizations.all()}" @@ -42,7 +42,6 @@ def __str__(self): User._meta.get_field("email")._unique = True User._meta.get_field("email").blank = False User._meta.get_field("email").null = False -User = get_user_model() class Organization(models.Model): diff --git a/core/schemas.py b/core/schemas.py index 07b369b..43097fe 100644 --- a/core/schemas.py +++ b/core/schemas.py @@ -8,7 +8,6 @@ from datetime import datetime from typing import Dict, Optional -from enum import Enum from django.contrib.auth.models import User from ninja import Field, ModelSchema, Schema from pydantic import UUID4 @@ -267,19 +266,10 @@ class SocialAccount(Schema): id_token: str -class UserRole(Enum): - ENGINEERING = 'engineering' - MARKETING = 'marketing' - OTHER = 'other' - -class UserMetaDetails(Schema): - role: UserRole - promotions: bool - class SocialUser(Schema): name: str email: str - meta: Optional[UserMetaDetails] + meta: Optional[dict] class SocialAuthLoginSchema(Schema): account: SocialAccount From 8fbe50b87e08826f5023cf3660d6a04f13d342f6 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Thu, 16 May 2024 19:16:23 +0530 Subject: [PATCH 078/159] feat: add migrations file --- Dockerfile | 6 +++--- core/migrations/0027_user_meta.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 core/migrations/0027_user_meta.py diff --git a/Dockerfile b/Dockerfile index 3291af0..fb50c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,10 @@ RUN pip install -r /tmp/requirements.txt WORKDIR /workspace -RUN groupadd -r valmi_group && useradd -r -g valmi_group valmi_user -RUN chown -R valmi_user:valmi_group /workspace +# RUN groupadd -r valmi_group && useradd -r -g valmi_group valmi_user +# RUN chown -R valmi_user:valmi_group /workspace -USER valmi_user +# USER valmi_user ENV PYTHONUNBUFFERED 1 diff --git a/core/migrations/0027_user_meta.py b/core/migrations/0027_user_meta.py new file mode 100644 index 0000000..a9a8b09 --- /dev/null +++ b/core/migrations/0027_user_meta.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-05-16 13:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0026_auto_20240503_0513'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='meta', + field=models.JSONField(default=dict), + ), + ] From 167faa51e5b7f7ed8fa8b58ee30529efdc015d62 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Fri, 17 May 2024 10:46:35 +0530 Subject: [PATCH 079/159] save: meta details & revert: Dockerfile changes --- Dockerfile | 6 +++--- core/social_auth.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb50c64..3291af0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,10 @@ RUN pip install -r /tmp/requirements.txt WORKDIR /workspace -# RUN groupadd -r valmi_group && useradd -r -g valmi_group valmi_user -# RUN chown -R valmi_user:valmi_group /workspace +RUN groupadd -r valmi_group && useradd -r -g valmi_group valmi_user +RUN chown -R valmi_user:valmi_group /workspace -# USER valmi_user +USER valmi_user ENV PYTHONUNBUFFERED 1 diff --git a/core/social_auth.py b/core/social_auth.py index aa57752..66b26ef 100644 --- a/core/social_auth.py +++ b/core/social_auth.py @@ -6,7 +6,6 @@ from rest_framework.authtoken.models import Token from core.schemas import DetailSchema, SocialAuthLoginSchema from core.models import User, Organization, Workspace, OAuthApiKeys -from core.services import warehouse_credentials import binascii import os import uuid @@ -39,13 +38,14 @@ def login(request, payload: SocialAuthLoginSchema): account = req["account"] email = user_data["email"] - + try: user = User.objects.get(email=email) except User.DoesNotExist: user = User() user.email = user_data["email"] user.username = user_data["name"] + user.meta = user_data['meta'] user.password = generate_key() user.save(force_insert=True) org = Organization(name="Default Organization", id=uuid.uuid4()) From 66a2712103b6bafb61a1905d5423c30a76b642a1 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 17 May 2024 13:27:28 +0530 Subject: [PATCH 080/159] feat: added try except block for preview data endpoint --- core/prompt_api.py | 58 ++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/core/prompt_api.py b/core/prompt_api.py index ca57dfc..2e0cebd 100644 --- a/core/prompt_api.py +++ b/core/prompt_api.py @@ -15,6 +15,7 @@ router = Router() + @router.get("/", response={200: List[PromptSchemaOut], 400: DetailSchema}) def get_prompts(request): try: @@ -22,22 +23,23 @@ def get_prompts(request): connector_ids = list(Credential.objects.values('connector_id').distinct()) connector_types = [connector['connector_id'] for connector in connector_ids] for prompt in prompts: - prompt["id"] = str(prompt["id"]) + prompt["id"] = str(prompt["id"]) if prompt["type"] in connector_types: prompt["enabled"] = True else: prompt["enabled"] = False return prompts except Exception as err: - logger.exception("prompts listing error:"+ err) + logger.exception("prompts listing error:" + err) return (400, {"detail": "The list of prompts cannot be fetched."}) + @router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptByIdSchema, 400: DetailSchema}) -def get_prompt(request,workspace_id,prompt_id): +def get_prompt(request, workspace_id, prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) - if not PromptService.is_prompt_enabled(workspace_id,prompt): + if not PromptService.is_prompt_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} credential_info = Source.objects.filter( @@ -69,29 +71,35 @@ def get_prompt(request,workspace_id,prompt_id): logger.exception("prompt listing error") return (400, {"detail": "The prompt cannot be fetched."}) - + def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() - -@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview",response={200: Json, 400: DetailSchema}) -def preview_data(request, workspace_id,prompt_id, prompt_req: PromptPreviewSchemaIn): - prompt = Prompt.objects.get(id=prompt_id) - if not PromptService.is_prompt_enabled(workspace_id,prompt): + + +@router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview", response={200: Json, 400: DetailSchema}) +def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSchemaIn): + try: + prompt = Prompt.objects.get(id=prompt_id) + if not PromptService.is_prompt_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} - storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) - schema_name = storage_credentials.connector_config["schema"] - table_name = f'{schema_name}.{prompt.table}' - query = PromptService().build(table_name, prompt_req.time_window, prompt_req.filters) - logger.debug(query) - host_url = os.environ["DATA_WAREHOUSE_URL"] - db_password = storage_credentials.connector_config.get('password') - db_username = storage_credentials.connector_config.get('username') - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) - cursor = conn.cursor() - cursor.execute(query) - items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] - conn.commit() - conn.close() - return json.dumps(items, indent=4, default=custom_serializer) + storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) + schema_name = storage_credentials.connector_config["schema"] + table_name = f'{schema_name}.{prompt.table}' + query = PromptService().build(table_name, prompt_req.time_window, prompt_req.filters) + logger.debug(query) + host_url = os.environ["DATA_WAREHOUSE_URL"] + db_password = storage_credentials.connector_config.get('password') + db_username = storage_credentials.connector_config.get('username') + conn = psycopg2.connect(host=host_url, port="5432", database="dvdrental", + user=db_username, password=db_password) + cursor = conn.cursor() + cursor.execute(query) + items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] + conn.commit() + conn.close() + return json.dumps(items, indent=4, default=custom_serializer) + except Exception as err: + logger.exception(f"preview fetching error:{err}") + return (400, {"detail": "Data cannot be fetched."}) From 5f834b47d6d54df3b40141053479615d6be018d3 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 17 May 2024 14:29:44 +0530 Subject: [PATCH 081/159] feat: Added missing imports in schema.py --- core/schemas/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 2bb0ae7..613040f 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -7,7 +7,7 @@ """ from datetime import datetime -from typing import Dict, Optional +from typing import Dict, List, Optional from django.contrib.auth.models import User from ninja import Field, ModelSchema, Schema from pydantic import UUID4 @@ -291,6 +291,7 @@ class SocialUser(Schema): email: str meta: Optional[dict] + class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser From 9adb18e212cbad8fbdaeb5391a3908225f99b362 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 17 May 2024 15:39:35 +0530 Subject: [PATCH 082/159] feat: resolved migration merge conflict --- core/migrations/0027_auto_20240508_0539.py | 5 +++++ core/migrations/0027_user_meta.py | 18 ------------------ 2 files changed, 5 insertions(+), 18 deletions(-) delete mode 100644 core/migrations/0027_user_meta.py diff --git a/core/migrations/0027_auto_20240508_0539.py b/core/migrations/0027_auto_20240508_0539.py index 3fc94d5..3864ece 100644 --- a/core/migrations/0027_auto_20240508_0539.py +++ b/core/migrations/0027_auto_20240508_0539.py @@ -10,6 +10,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='user', + name='meta', + field=models.JSONField(default=dict), + ), migrations.RenameField( model_name='prompt', old_name='parameters', diff --git a/core/migrations/0027_user_meta.py b/core/migrations/0027_user_meta.py deleted file mode 100644 index a9a8b09..0000000 --- a/core/migrations/0027_user_meta.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.5 on 2024-05-16 13:45 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0026_auto_20240503_0513'), - ] - - operations = [ - migrations.AddField( - model_name='user', - name='meta', - field=models.JSONField(default=dict), - ), - ] From 16c75e318e9b67455e94b77e3da7005f2c5a26ff Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 21 May 2024 11:02:23 +0530 Subject: [PATCH 083/159] feat: Add Shopify source creation in /syncs/create endpoint --- core/api.py | 54 ++++++++++++++++++++++++++++++++++------- core/schemas/schemas.py | 7 ++++-- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/core/api.py b/core/api.py index f3403df..0e238ff 100644 --- a/core/api.py +++ b/core/api.py @@ -328,17 +328,53 @@ def create_destination(request, workspace_id, payload: DestinationSchemaIn): def create_sync(request, workspace_id, payload: SyncSchemaIn): data = payload.dict() try: - logger.debug(dict) - logger.debug(payload.source_id) - logger.debug(payload.destination_id) - data["id"] = uuid.uuid4() - data["workspace"] = Workspace.objects.get(id=workspace_id) - data["source"] = Source.objects.get(id=payload.source_id) - data["destination"] = Destination.objects.get(id=payload.destination_id) - + if data["source_id"] is None and data["destination_id"] is None: + source_config = data["source_config"] + logger.debug("source config is ") + catalog = data["source_catalog"] + for stream in catalog["streams"]: + primary_key = [["id"]] + stream["primary_key"] = primary_key + stream["destination_sync_mode"] = "append_dedup" + # creating source credential + source_credential_payload = CredentialSchemaIn( + name=source_config["name"], account=data["account"], connector_type=source_config["source_connector_type"], + connector_config=source_config["source_connector_config"]) + source_credential = create_credential(request, workspace_id, source_credential_payload) + # creating source + source_payload = SourceSchemaIn( + name="shopify", credential_id=source_credential.id, catalog=catalog) + source = create_source(request, workspace_id, source_payload) + workspace = Workspace.objects.get(id=workspace_id) + # creating default warehouse + storage_credentials = DefaultWarehouse.create(workspace) + source_access_info = {"source": source, "storage_credentials": storage_credentials} + SourceAccessInfo.objects.create(**source_access_info) + # creating destination credential + destination_credential_payload = CredentialSchemaIn( + name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + destination_credential = create_credential(request, workspace_id, destination_credential_payload) + # creating destination + destination_payload = DestinationSchemaIn( + name="default warehouse", credential_id=destination_credential.id, catalog=catalog) + destination = create_destination(request, workspace_id, destination_payload) + data["source_id"] = source.id + data["destination_id"] = destination.id + del data["source_config"] + del data["account"] + del data["source_catalog"] + + data["source"] = Source.objects.get(id=data["source_id"]) + data["destination"] = Destination.objects.get(id=data["destination_id"]) del data["source_id"] del data["destination_id"] - logger.debug(data["schedule"]) + schedule = {} + if len(data["schedule"]) == 0: + schedule["run_interval"] = 3600000 + data["schedule"] = schedule + data["workspace"] = Workspace.objects.get(id=workspace_id) + data["id"] = uuid.uuid4() + logger.debug(data) sync = Sync.objects.create(**data) return sync except Exception: diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 613040f..6b2e87c 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -203,9 +203,12 @@ class Config(CamelSchemaConfig): class SyncSchemaIn(Schema): name: str - source_id: UUID4 - destination_id: UUID4 + source_id:Optional[UUID4] = None + destination_id: Optional[UUID4] = None schedule: Dict + account: Optional[Dict] + source_config: Optional[Dict] + source_catalog: Optional[Dict] ui_state: Optional[Dict] From b639f3e2d4339b836ef9688c66a004298c46c68b Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 21 May 2024 12:26:38 +0530 Subject: [PATCH 084/159] feat: Remove query from ExploreSchemaInput --- core/explore_api.py | 11 +++++------ core/schemas/schemas.py | 6 ++++-- core/services/explore_service.py | 25 +++++++++++++++++++------ core/services/prompts.py | 24 +++++++++++++++--------- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/core/explore_api.py b/core/explore_api.py index 5b57e2d..0f67ee0 100644 --- a/core/explore_api.py +++ b/core/explore_api.py @@ -1,7 +1,6 @@ import json import logging import json -import time from typing import List import uuid from decouple import config @@ -45,8 +44,8 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): account = Account.objects.create(**account_info) data["account"] = account # create source - source = ExploreService.create_source(data["schema_id"], data["query"], workspace_id, account) - time.sleep(6) + source = ExploreService.create_source( + data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) # create destination spreadsheet_name = f"valmiio {prompt.name} sheet" destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) @@ -54,17 +53,17 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): destination = destination_data[1] # create sync sync = ExploreService.create_sync(source, destination, workspace_id) - # await asyncio.sleep(5) - time.sleep(5) # creating explore del data["schema_id"] - del data["query"] + del data["filters"] + del data["time_window"] data["name"] = f"valmiio {prompt.name}" data["sync"] = sync data["ready"] = False data["spreadsheet_url"] = spreadsheet_url explore = Explore.objects.create(**data) # create run + ExploreService.wait_for_run(5) payload = SyncStartStopSchemaIn(full_refresh=True) ExploreService.create_run(request, workspace_id, sync.id, payload) return explore diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 6b2e87c..ddf88c5 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -12,6 +12,7 @@ from ninja import Field, ModelSchema, Schema from pydantic import UUID4 from core.models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys +from core.schemas.prompt import Filter, TimeWindow def camel_to_snake(s): @@ -124,7 +125,8 @@ class ExploreSchemaIn(Schema): account: Dict = None prompt_id: str schema_id: str - query: str + time_window: TimeWindow + filters: list[Filter] class ExplorePreviewDataIn(Schema): @@ -203,7 +205,7 @@ class Config(CamelSchemaConfig): class SyncSchemaIn(Schema): name: str - source_id:Optional[UUID4] = None + source_id: Optional[UUID4] = None destination_id: Optional[UUID4] = None schedule: Dict account: Optional[Dict] diff --git a/core/services/explore_service.py b/core/services/explore_service.py index a507994..b7651fb 100644 --- a/core/services/explore_service.py +++ b/core/services/explore_service.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import os @@ -9,7 +10,9 @@ from decouple import config import requests from core.api import create_new_run -from core.models import Credential, Destination, OAuthApiKeys, Source, StorageCredentials, Sync, Workspace +from core.models import Credential, Destination, OAuthApiKeys, Prompt, Source, StorageCredentials, Sync, Workspace +from core.schemas.prompt import Filter, TimeWindow +from core.services.prompts import PromptService logger = logging.getLogger(__name__) ACTIVATION_URL = config("ACTIVATION_SERVER") SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] @@ -58,7 +61,7 @@ def create_spreadsheet(name: str, refresh_token: str) -> str: raise Exception("spreadhseet creation failed") @staticmethod - def create_source(schema_id: str, query: str, workspace_id: str, account: object) -> object: + def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filters: list[Filter], workspace_id: str, account: object) -> object: try: # creating source credentail credential = {"id": uuid.uuid4()} @@ -86,6 +89,13 @@ def create_source(schema_id: str, query: str, workspace_id: str, account: object # creating source object source["workspace"] = Workspace.objects.get(id=workspace_id) source["credential"] = Credential.objects.get(id=cred.id) + # building query + logger.debug(type(time_window)) + prompt = Prompt.objects.get(id=prompt_id) + namespace = storage_credential.connector_config["namespace"] + table = f'{namespace}.{prompt.table}' + query = PromptService().build(table, time_window, filters) + # creating source cayalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" body = { "ssl": False, @@ -107,12 +117,11 @@ def create_source(schema_id: str, query: str, workspace_id: str, account: object json_file_path = join(dirname(__file__), 'source_catalog.json') with open(json_file_path, 'r') as openfile: source_catalog = json.load(openfile) - # TODO : HARDCODED TABLE NAME source_catalog["streams"][0]["stream"] = response_json["catalog"]["streams"][0] - namespace = storage_credential.connector_config["namespace"] database = storage_credential.connector_config["database"] - table = "orders_with_product_data" - source_catalog["streams"][0]["stream"]["name"] = f"{database}.{namespace}.{table}" + source_catalog["streams"][0]["stream"][ + "name" + ] = f"{database}.{namespace}.{table}" source["catalog"] = source_catalog source["status"] = "active" logger.debug(source_catalog) @@ -188,6 +197,10 @@ def create_sync(source: object, destination: object, workspace_id: str) -> objec logger.exception(f"Error creating sync: {e}") raise Exception("unable to create sync") + @staticmethod + async def wait_for_run(time: int): + await asyncio.sleep(time) + @staticmethod def create_run(request: object, workspace_id: str, sync_id: str, payload: object) -> None: try: diff --git a/core/services/prompts.py b/core/services/prompts.py index 4f7525d..d8cfbac 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -7,15 +7,24 @@ from core.schemas.prompt import TimeWindow, Filter logger = logging.getLogger(__name__) -class PromptService(): + + +class PromptService(): @classmethod def getTemplateFile(cls): return 'prompts.liquid' + @staticmethod - def build(table, timeWindow: TimeWindow , filters: list[Filter]) -> str: - timeWindowDict = timeWindow.dict() - filterList = [filter.__dict__ for filter in filters] - file_name = PromptService.getTemplateFile() + def build(table, timeWindow: TimeWindow, filters: list[Filter]) -> str: + if isinstance(timeWindow, TimeWindow): + timeWindowDict = timeWindow.dict() + else: + timeWindowDict = timeWindow + if isinstance(filters, Filter): + filterList = [filter.__dict__ for filter in filters] + else: + filterList = filters + file_name = PromptService.getTemplateFile() template_parent_path = Path(__file__).parent.absolute() env = Environment( tolerance=Mode.STRICT, @@ -29,10 +38,7 @@ def build(table, timeWindow: TimeWindow , filters: list[Filter]) -> str: return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) @staticmethod - def is_prompt_enabled(workspace_id:str,prompt:object) -> bool: + def is_prompt_enabled(workspace_id: str, prompt: object) -> bool: connector_ids = list(Credential.objects.filter(workspace_id=workspace_id).values('connector_id').distinct()) connector_types = [connector['connector_id'] for connector in connector_ids] return prompt.type in connector_types - - - \ No newline at end of file From 06144081864d4dec1080e3ae0921d009844be006 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 21 May 2024 15:57:04 +0530 Subject: [PATCH 085/159] feat: Add endpoint for sync with defaults --- core/api.py | 82 +++++++++++++++++++++++------------------ core/schemas/schemas.py | 13 +++++-- 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/core/api.py b/core/api.py index 0e238ff..9917538 100644 --- a/core/api.py +++ b/core/api.py @@ -34,6 +34,7 @@ SyncIdSchema, SyncSchema, SyncSchemaIn, + SyncSchemaInWithSourcePayload, SyncSchemaUpdateIn, SyncStartStopSchemaIn, UserSchemaOut, @@ -328,42 +329,6 @@ def create_destination(request, workspace_id, payload: DestinationSchemaIn): def create_sync(request, workspace_id, payload: SyncSchemaIn): data = payload.dict() try: - if data["source_id"] is None and data["destination_id"] is None: - source_config = data["source_config"] - logger.debug("source config is ") - catalog = data["source_catalog"] - for stream in catalog["streams"]: - primary_key = [["id"]] - stream["primary_key"] = primary_key - stream["destination_sync_mode"] = "append_dedup" - # creating source credential - source_credential_payload = CredentialSchemaIn( - name=source_config["name"], account=data["account"], connector_type=source_config["source_connector_type"], - connector_config=source_config["source_connector_config"]) - source_credential = create_credential(request, workspace_id, source_credential_payload) - # creating source - source_payload = SourceSchemaIn( - name="shopify", credential_id=source_credential.id, catalog=catalog) - source = create_source(request, workspace_id, source_payload) - workspace = Workspace.objects.get(id=workspace_id) - # creating default warehouse - storage_credentials = DefaultWarehouse.create(workspace) - source_access_info = {"source": source, "storage_credentials": storage_credentials} - SourceAccessInfo.objects.create(**source_access_info) - # creating destination credential - destination_credential_payload = CredentialSchemaIn( - name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) - destination_credential = create_credential(request, workspace_id, destination_credential_payload) - # creating destination - destination_payload = DestinationSchemaIn( - name="default warehouse", credential_id=destination_credential.id, catalog=catalog) - destination = create_destination(request, workspace_id, destination_payload) - data["source_id"] = source.id - data["destination_id"] = destination.id - del data["source_config"] - del data["account"] - del data["source_catalog"] - data["source"] = Source.objects.get(id=data["source_id"]) data["destination"] = Destination.objects.get(id=data["destination_id"]) del data["source_id"] @@ -382,6 +347,51 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): return {"detail": "The specific sync cannot be created."} +@router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 400: DetailSchema}) +def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): + data = payload.dict() + source_config = data["source"]["config"] + catalog = data["source"]["catalog"] + for stream in catalog["streams"]: + primary_key = [["id"]] + stream["primary_key"] = primary_key + stream["destination_sync_mode"] = "append_dedup" + # creating source credential + source_credential_payload = CredentialSchemaIn( + name=source_config["name"], account=data["account"], connector_type=source_config["source_connector_type"], + connector_config=source_config["source_connector_config"]) + source_credential = create_credential(request, workspace_id, source_credential_payload) + # creating source + source_payload = SourceSchemaIn( + name="shopify", credential_id=source_credential.id, catalog=catalog) + source = create_source(request, workspace_id, source_payload) + workspace = Workspace.objects.get(id=workspace_id) + # creating default warehouse + storage_credentials = DefaultWarehouse.create(workspace) + source_access_info = {"source": source, "storage_credentials": storage_credentials} + SourceAccessInfo.objects.create(**source_access_info) + # creating destination credential + destination_credential_payload = CredentialSchemaIn( + name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + destination_credential = create_credential(request, workspace_id, destination_credential_payload) + # creating destination + destination_payload = DestinationSchemaIn( + name="default warehouse", credential_id=destination_credential.id, catalog=catalog) + destination = create_destination(request, workspace_id, destination_payload) + data["source"] = Source.objects.get(id=source.id) + data["destination"] = Destination.objects.get(id=destination.id) + del data["account"] + schedule = {} + if len(data["schedule"]) == 0: + schedule["run_interval"] = 3600000 + data["schedule"] = schedule + data["workspace"] = Workspace.objects.get(id=workspace_id) + data["id"] = uuid.uuid4() + logger.debug(data) + sync = Sync.objects.create(**data) + return sync + + @router.post("/workspaces/{workspace_id}/syncs/update", response={200: SyncSchema, 400: DetailSchema}) def update_sync(request, workspace_id, payload: SyncSchemaUpdateIn): data = payload.dict() diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index ddf88c5..97cad6d 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -205,12 +205,17 @@ class Config(CamelSchemaConfig): class SyncSchemaIn(Schema): name: str - source_id: Optional[UUID4] = None - destination_id: Optional[UUID4] = None + source_id: UUID4 + destination_id: UUID4 schedule: Dict + ui_state: Optional[Dict] + + +class SyncSchemaInWithSourcePayload(Schema): + name: str + source: Dict account: Optional[Dict] - source_config: Optional[Dict] - source_catalog: Optional[Dict] + schedule: Dict ui_state: Optional[Dict] From 6bbc95b9be6649bd7f06f0ba5238aef4484ea9ea Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Tue, 21 May 2024 16:45:28 +0530 Subject: [PATCH 086/159] feat: refactor api routes (#35) * feat: refactor api routes --- core/routes/__init__.py | 0 core/routes/api.py | 152 +++++++++++++++++++++ core/routes/api_config.py | 2 + core/routes/connector_api.py | 107 +++++++++++++++ core/{ => routes}/engine_api.py | 2 +- core/{ => routes}/explore_api.py | 26 ++-- core/{ => routes}/oauth_api.py | 2 +- core/{ => routes}/package_api.py | 0 core/{ => routes}/prompt_api.py | 8 +- core/{ => routes}/social_auth.py | 0 core/{ => routes}/stream_api.py | 9 +- core/{api.py => routes/workspace_api.py} | 164 +++-------------------- core/services/explore_service.py | 12 +- valmi_app_backend/urls.py | 121 +---------------- 14 files changed, 313 insertions(+), 292 deletions(-) create mode 100644 core/routes/__init__.py create mode 100644 core/routes/api.py create mode 100644 core/routes/api_config.py create mode 100644 core/routes/connector_api.py rename core/{ => routes}/engine_api.py (99%) rename core/{ => routes}/explore_api.py (90%) rename core/{ => routes}/oauth_api.py (98%) rename core/{ => routes}/package_api.py (100%) rename core/{ => routes}/prompt_api.py (78%) rename core/{ => routes}/social_auth.py (100%) rename core/{ => routes}/stream_api.py (98%) rename core/{api.py => routes/workspace_api.py} (79%) diff --git a/core/routes/__init__.py b/core/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/routes/api.py b/core/routes/api.py new file mode 100644 index 0000000..88558e8 --- /dev/null +++ b/core/routes/api.py @@ -0,0 +1,152 @@ +""" +Copyright (c) 2024 valmi.io + +Created Date: Wednesday, March 8th 2023, 9:56:52 pm +Author: Rajashekar Varkala @ valmi.io + +""" + +import logging +from typing import Any, Optional + +from decouple import config +from django.conf import settings +from django.http import HttpRequest +from ninja import NinjaAPI, Router +from ninja.compatibility import get_headers +from ninja.security import HttpBasicAuth, HttpBearer +from rest_framework.authentication import BasicAuthentication + +from core.routes.connector_api import router as connector_api_router +from core.routes.engine_api import router as superuser_api_router +from core.routes.explore_api import router as explore_api_router +from core.routes.oauth_api import router as oauth_api_router +from core.routes.package_api import router as package_api_router +from core.routes.prompt_api import router as prompt_api_router +from core.routes.social_auth import router as social_api_router +from core.routes.stream_api import router as stream_api_router +from core.routes.workspace_api import router as workspace_api_router +from valmi_app_backend.utils import BearerAuthentication + +from ..models import User + +# Get an instance of a logger +logger = logging.getLogger(__name__) + +from core.schemas import UserSchemaOut + +auth_enabled = config("AUTHENTICATION", default=True, cast=bool) +class BasicAuth(HttpBasicAuth): + def authenticate(self, request, username, password): + if auth_enabled == False: + return True + try: + user_auth_tuple = BasicAuthentication().authenticate(request) + except Exception: + user_auth_tuple = None + if user_auth_tuple is not None: + (user, token) = user_auth_tuple + request.user = user + # Basic Auth allowed only for superuser. + if user.is_superuser: + return user + return None + return None + + +class AuthBearer(HttpBearer): + openapi_scheme: str = "bearer" + + def __call__(self, request: HttpRequest) -> Optional[Any]: + return self.authenticate(request) + + def has_permission_for(self, user, workspace_id): + if auth_enabled == False: + return True + for workspace in get_workspaces(user): + logger.debug("checking workspace %s", workspace.id) + if str(workspace.id) == workspace_id: + return True + return False + + def authenticate(self, request): + ''' + logger.debug("enabled " +config('PUBLIC_SYNC_ENABLED')) + logger.debug("pub " +config('PUBLIC_WORKSPACE')) + logger.debug("sync " +config('PUBLIC_SYNC')) + logger.debug("authtoken " +config('PUBLIC_AUTH_TOKEN')) + logger.debug("path "+ request.path) + logger.debug("hardcoded "+ f'/api/v1/workspaces/{config("PUBLIC_WORKSPACE")}/syncs/{config("PUBLIC_SYNC")}/runs/') + ''' + if (config('PUBLIC_SYNC_ENABLED', default=False) and + request.path == f'/api/v1/workspaces/{config("PUBLIC_WORKSPACE")}/syncs/{config("PUBLIC_SYNC")}/runs/'): + return config('PUBLIC_AUTH_TOKEN') + + headers = get_headers(request) + auth_value = headers.get(self.header) + if not auth_value: + return None + parts = auth_value.split(" ") + + if parts[0].lower() != self.openapi_scheme: + if settings.DEBUG: + logger.error(f"Unexpected auth - '{auth_value}'") + return None + token = " ".join(parts[1:]) + + try: + user_auth_tuple = BearerAuthentication().authenticate(request) + except Exception: + user_auth_tuple = None + if user_auth_tuple is not None: + (user, token) = user_auth_tuple # here come your user object + request.user = user + # get Workspace Id. + arr = request.get_full_path().split("/") + for i, el in enumerate(arr): + if el == "workspaces" and len(arr) > i + 1: + workspace_id = arr[i + 1] + logger.debug(workspace_id) + if not self.has_permission_for(user, workspace_id): + return None + break + return token + return None + +def get_workspaces(user): + queryset = User.objects.prefetch_related("organizations").get(id=user.id) + for organization in queryset.organizations.all(): + for workspace in organization.workspaces.all(): + yield workspace + + +router = Router() +@router.get("/spaces/", response=UserSchemaOut) +def list_spaces(request): + user_id = request.user.id + queryset = User.objects.prefetch_related("organizations").get(id=user_id) + logger.debug(queryset) + return queryset + + + + +api = NinjaAPI( + version="1.0", + csrf=False, + title="Valmi App Backend API", + description="App Backend API Serves the Valmi App Frontend", + urls_namespace="public_api", +) + +api.add_router("/auth/social", social_api_router) + +api.add_router("v1/", router, auth=[AuthBearer(),BasicAuth()]) +router.add_router("superuser/", superuser_api_router, auth=[BasicAuth()]) +router.add_router("streams/",stream_api_router, tags = ["streams"]) +router.add_router("oauth/", oauth_api_router, tags=["oauth"]) +router.add_router("connectors", connector_api_router, tags=["connectors"]) +router.add_router("packages", package_api_router, tags = ["packages"]) +router.add_router("", workspace_api_router, tags=["workspaces"]) +router.add_router("", prompt_api_router, tags = ["prompts"]) +router.add_router("", explore_api_router, tags = ["explores"]) diff --git a/core/routes/api_config.py b/core/routes/api_config.py new file mode 100644 index 0000000..5414286 --- /dev/null +++ b/core/routes/api_config.py @@ -0,0 +1,2 @@ +LONG_TIMEOUT = 60 +SHORT_TIMEOUT = 60 \ No newline at end of file diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py new file mode 100644 index 0000000..ad43294 --- /dev/null +++ b/core/routes/connector_api.py @@ -0,0 +1,107 @@ +import logging +from typing import Dict, List + +from ninja import Router + +from core.schemas import ConnectorSchema, DetailSchema + +from ..models import Connector, OAuthApiKeys, Workspace + +router = Router() + +# Get an instance of a logger +logger = logging.getLogger(__name__) + +@router.get("/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) +def get_connectors(request): + # check for admin permissions + try: + logger.debug("listing connectors") + connectors = Connector.objects.all() + + logger.info(f"connectors - {connectors}") + src_dst_dict: Dict[str, List[ConnectorSchema]] = {} + src_dst_dict["SRC"] = [] + src_dst_dict["DEST"] = [] + for conn in connectors: + logger.info(f"conn{conn}") + arr = conn.type.split("_") + if arr[0] == "SRC": + src_dst_dict["SRC"].append(conn) + elif arr[0] == "DEST": + src_dst_dict["DEST"].append(conn) + return src_dst_dict + except Exception: + logger.exception("connector listing error") + return (400, {"detail": "The list of connectors cannot be fetched."}) + + +@router.get("{workspace_id}/configured", response={200: Dict[str, List[ConnectorSchema]], + 400: DetailSchema}) +def get_connectors_configured(request, workspace_id): + + try: + # Get the connectors that match the criteria + logger.debug("listing all configured connectors") + workspace = Workspace.objects.get(id=workspace_id) + + configured_connectors = OAuthApiKeys.objects.filter(workspace=workspace).values('type') + + connectors = Connector.objects.filter( + oauth=True, + oauth_keys="private", + type__in=configured_connectors) + + src_dst_dict: Dict[str, List[ConnectorSchema]] = {} + src_dst_dict["SRC"] = [] + src_dst_dict["DEST"] = [] + logger.debug("Connectors:-", connectors) + for conn in connectors: + arr = conn.type.split("_") + logger.debug("Arr:_", arr) + if arr[0] == "SRC": + src_dst_dict["SRC"].append(conn) + elif arr[0] == "DEST": + src_dst_dict["DEST"].append(conn) + + return src_dst_dict + + except Workspace.DoesNotExist: + return (400, {"detail": "Workspace not found."}) + + except Exception: + logger.exception("connector listing error") + return (400, {"detail": "The list of connectors cannot be fetched."}) + + +@router.get("{workspace_id}/not-configured", + response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) +def get_connectors_not_configured(request, workspace_id): + + try: + # Get the connectors that match the criteria + + workspace = Workspace.objects.get(id=workspace_id) + + configured_connectors = OAuthApiKeys.objects.filter(workspace=workspace).values('type') + + connectors = Connector.objects.filter( + oauth=True, + oauth_keys="private" + ).exclude(type__in=configured_connectors) + + src_dst_dict: Dict[str, List[ConnectorSchema]] = {} + src_dst_dict["SRC"] = [] + src_dst_dict["DEST"] = [] + logger.debug("Connectors:-", connectors) + for conn in connectors: + arr = conn.type.split("_") + if arr[0] == "SRC": + src_dst_dict["SRC"].append(conn) + elif arr[0] == "DEST": + src_dst_dict["DEST"].append(conn) + return src_dst_dict + + except Exception: + logger.exception("connector listing error") + return (400, {"detail": "The list of connectors cannot be fetched."}) \ No newline at end of file diff --git a/core/engine_api.py b/core/routes/engine_api.py similarity index 99% rename from core/engine_api.py rename to core/routes/engine_api.py index a18d61a..2cfbc7a 100644 --- a/core/engine_api.py +++ b/core/routes/engine_api.py @@ -14,7 +14,7 @@ from core.schemas import ConnectorSchema, DetailSchema, PackageSchema, PromptSchema, SyncSchema -from .models import ( +from ..models import ( Connector, Package, Prompt, diff --git a/core/explore_api.py b/core/routes/explore_api.py similarity index 90% rename from core/explore_api.py rename to core/routes/explore_api.py index 60c3134..3c0c578 100644 --- a/core/explore_api.py +++ b/core/routes/explore_api.py @@ -1,28 +1,28 @@ import datetime import json import logging -import json import os -from typing import List +import time import uuid +from typing import List + +import psycopg2 +import requests from decouple import config -import time -import json +from ninja import Router from pydantic import Json -import requests - -import psycopg2 from core.models import Account, Explore, Prompt, StorageCredentials, Workspace -from core.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn -from ninja import Router +from core.schemas import (DetailSchema, ExploreSchema, ExploreSchemaIn, + SyncStartStopSchemaIn) from core.services.explore_service import ExploreService + logger = logging.getLogger(__name__) router = Router() ACTIVATION_URL = config("ACTIVATION_SERVER") -@router.get("/workspaces/{workspace_id}", response={200: List[ExploreSchema], 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchema], 400: DetailSchema}) def get_explores(request,workspace_id): try: logger.debug("listing explores") @@ -33,7 +33,7 @@ def get_explores(request,workspace_id): return (400, {"detail": "The list of explores cannot be fetched."}) -@router.post("/workspaces/{workspace_id}/create",response={200: ExploreSchema, 400: DetailSchema}) +@router.post("/workspaces/{workspace_id}/explores/create",response={200: ExploreSchema, 400: DetailSchema}) def create_explore(request, workspace_id,payload: ExploreSchemaIn): data = payload.dict() try: @@ -105,7 +105,7 @@ def preview_data(request, workspace_id,prompt_id): return json.dumps(items, indent=4, default=custom_serializer) -@router.get("/workspaces/{workspace_id}/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/explores/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) def get_explores(request,workspace_id,explore_id): try: logger.debug("listing explores") @@ -115,7 +115,7 @@ def get_explores(request,workspace_id,explore_id): return (400, {"detail": "The explore cannot be fetched."}) -@router.get("/workspaces/{workspace_id}/{explore_id}/status", response={200: Json, 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/explores/{explore_id}/status", response={200: Json, 400: DetailSchema}) def get_explore_status(request,workspace_id,explore_id): try: logger.debug("getting_explore_status") diff --git a/core/oauth_api.py b/core/routes/oauth_api.py similarity index 98% rename from core/oauth_api.py rename to core/routes/oauth_api.py index 84cc813..ef92e2a 100644 --- a/core/oauth_api.py +++ b/core/routes/oauth_api.py @@ -16,7 +16,7 @@ from ninja import Router from pydantic import Json -from .models import Connector, Workspace, OAuthApiKeys +from ..models import Connector, Workspace, OAuthApiKeys from core.schemas import ( DetailSchema, diff --git a/core/package_api.py b/core/routes/package_api.py similarity index 100% rename from core/package_api.py rename to core/routes/package_api.py diff --git a/core/prompt_api.py b/core/routes/prompt_api.py similarity index 78% rename from core/prompt_api.py rename to core/routes/prompt_api.py index 008a265..ca31786 100644 --- a/core/prompt_api.py +++ b/core/routes/prompt_api.py @@ -1,15 +1,15 @@ import logging from typing import List +from ninja import Router + from core.models import Prompt from core.schemas import DetailSchema, PromptSchema -from ninja import Router logger = logging.getLogger(__name__) - router = Router() -@router.get("/", response={200: List[PromptSchema], 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchema], 400: DetailSchema}) def get_prompts(request): try: logger.debug("listing prompts") @@ -19,7 +19,7 @@ def get_prompts(request): logger.exception("prompts listing error") return (400, {"detail": "The list of prompts cannot be fetched."}) -@router.get("/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) def get_prompts(request,prompt_id): try: logger.debug("listing prompts") diff --git a/core/social_auth.py b/core/routes/social_auth.py similarity index 100% rename from core/social_auth.py rename to core/routes/social_auth.py diff --git a/core/stream_api.py b/core/routes/stream_api.py similarity index 98% rename from core/stream_api.py rename to core/routes/stream_api.py index f9d8724..b82bf99 100644 --- a/core/stream_api.py +++ b/core/routes/stream_api.py @@ -8,16 +8,17 @@ import json import logging +from typing import Optional import requests +from decouple import config from ninja import Router from pydantic import Json -from decouple import config -from core.api import SHORT_TIMEOUT + +from core.routes.api_config import SHORT_TIMEOUT from core.schemas import GenericJsonSchema -from .models import ValmiUserIDJitsuApiToken -from typing import Optional +from ..models import ValmiUserIDJitsuApiToken router = Router() diff --git a/core/api.py b/core/routes/workspace_api.py similarity index 79% rename from core/api.py rename to core/routes/workspace_api.py index dff1155..13d849d 100644 --- a/core/api.py +++ b/core/routes/workspace_api.py @@ -1,76 +1,43 @@ -""" -Copyright (c) 2024 valmi.io - -Created Date: Wednesday, March 8th 2023, 9:56:52 pm -Author: Rajashekar Varkala @ valmi.io - -""" - import json import logging import uuid -import uuid from datetime import datetime from typing import Dict, List, Optional + import requests from decouple import Csv, config from ninja import Router from pydantic import UUID4, Json -from core.schemas import ( - ConnectionSchemaIn, - ConnectorConfigSchemaIn, - ConnectorSchema, - CredentialSchema, - CredentialSchemaIn, - CredentialSchemaUpdateIn, - DestinationSchema, - DestinationSchemaIn, - DetailSchema, - FailureSchema, - SourceSchema, - SourceSchemaIn, - SuccessSchema, - SyncIdSchema, - SyncSchema, - SyncSchemaIn, - SyncSchemaUpdateIn, - SyncStartStopSchemaIn, - UserSchemaOut, - CreateConfigSchemaIn -) -from .models import Account, Connector, Credential, Destination, Source, StorageCredentials, Sync, User, Workspace, OAuthApiKeys -from valmi_app_backend.utils import replace_values_in_json + +from core.routes.api_config import LONG_TIMEOUT, SHORT_TIMEOUT +from core.schemas import (ConnectionSchemaIn, ConnectorConfigSchemaIn, + CreateConfigSchemaIn, CredentialSchema, + CredentialSchemaIn, CredentialSchemaUpdateIn, + DestinationSchema, DestinationSchemaIn, DetailSchema, + FailureSchema, SourceSchema, SourceSchemaIn, + SuccessSchema, SyncIdSchema, SyncSchema, + SyncSchemaIn, SyncSchemaUpdateIn, + SyncStartStopSchemaIn) from core.services import warehouse_credentials +from valmi_app_backend.utils import replace_values_in_json + +from ..models import (Account, Connector, Credential, Destination, + OAuthApiKeys, Source, StorageCredentials, Sync, + Workspace) router = Router() +# Get an instance of a logger +logger = logging.getLogger(__name__) + CONNECTOR_PREFIX_URL = config("ACTIVATION_SERVER") + "/connectors" ACTIVATION_URL = config("ACTIVATION_SERVER") ACTIVE = "active" INACTIVE = "inactive" DELETED = "deleted" -LONG_TIMEOUT = 60 -SHORT_TIMEOUT = 60 - -# Get an instance of a logger -logger = logging.getLogger(__name__) -def get_workspaces(user): - queryset = User.objects.prefetch_related("organizations").get(id=user.id) - for organization in queryset.organizations.all(): - for workspace in organization.workspaces.all(): - yield workspace - - -@router.get("/spaces/", response=UserSchemaOut) -def list_spaces(request): - user_id = request.user.id - queryset = User.objects.prefetch_related("organizations").get(id=user_id) - logger.debug(queryset) - return queryset - @router.get("/workspaces/{workspace_id}/connectors/{connector_type}/spec", response=Json) def connector_spec(request, workspace_id, connector_type): @@ -464,30 +431,6 @@ def get_run(request, workspace_id, sync_id: UUID4, run_id: UUID4): ).text -@router.get("/connectors/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) -def get_connectors(request): - # check for admin permissions - try: - logger.debug("listing connectors") - connectors = Connector.objects.all() - - logger.info(f"connectors - {connectors}") - src_dst_dict: Dict[str, List[ConnectorSchema]] = {} - src_dst_dict["SRC"] = [] - src_dst_dict["DEST"] = [] - for conn in connectors: - logger.info(f"conn{conn}") - arr = conn.type.split("_") - if arr[0] == "SRC": - src_dst_dict["SRC"].append(conn) - elif arr[0] == "DEST": - src_dst_dict["DEST"].append(conn) - return src_dst_dict - except Exception: - logger.exception("connector listing error") - return (400, {"detail": "The list of connectors cannot be fetched."}) - - @router.get("/workspaces/{workspace_id}/syncs/{sync_id}/runs/{run_id}/logs", response=Json) def get_logs( request, @@ -519,72 +462,3 @@ def get_samples( ).text -@router.get("/connectors/{workspace_id}/configured", response={200: Dict[str, List[ConnectorSchema]], - 400: DetailSchema}) -def get_connectors_configured(request, workspace_id): - - try: - # Get the connectors that match the criteria - logger.debug("listing all configured connectors") - workspace = Workspace.objects.get(id=workspace_id) - - configured_connectors = OAuthApiKeys.objects.filter(workspace=workspace).values('type') - - connectors = Connector.objects.filter( - oauth=True, - oauth_keys="private", - type__in=configured_connectors) - - src_dst_dict: Dict[str, List[ConnectorSchema]] = {} - src_dst_dict["SRC"] = [] - src_dst_dict["DEST"] = [] - logger.debug("Connectors:-", connectors) - for conn in connectors: - arr = conn.type.split("_") - logger.debug("Arr:_", arr) - if arr[0] == "SRC": - src_dst_dict["SRC"].append(conn) - elif arr[0] == "DEST": - src_dst_dict["DEST"].append(conn) - - return src_dst_dict - - except Workspace.DoesNotExist: - return (400, {"detail": "Workspace not found."}) - - except Exception: - logger.exception("connector listing error") - return (400, {"detail": "The list of connectors cannot be fetched."}) - - -@router.get("/connectors/{workspace_id}/not-configured", - response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) -def get_connectors_not_configured(request, workspace_id): - - try: - # Get the connectors that match the criteria - - workspace = Workspace.objects.get(id=workspace_id) - - configured_connectors = OAuthApiKeys.objects.filter(workspace=workspace).values('type') - - connectors = Connector.objects.filter( - oauth=True, - oauth_keys="private" - ).exclude(type__in=configured_connectors) - - src_dst_dict: Dict[str, List[ConnectorSchema]] = {} - src_dst_dict["SRC"] = [] - src_dst_dict["DEST"] = [] - logger.debug("Connectors:-", connectors) - for conn in connectors: - arr = conn.type.split("_") - if arr[0] == "SRC": - src_dst_dict["SRC"].append(conn) - elif arr[0] == "DEST": - src_dst_dict["DEST"].append(conn) - return src_dst_dict - - except Exception: - logger.exception("connector listing error") - return (400, {"detail": "The list of connectors cannot be fetched."}) diff --git a/core/services/explore_service.py b/core/services/explore_service.py index 48c1ca0..76785c4 100644 --- a/core/services/explore_service.py +++ b/core/services/explore_service.py @@ -1,13 +1,17 @@ import json import logging import os -from typing import List, Union import uuid +from os.path import dirname, join +from typing import List, Union + from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from os.path import dirname, join -from core.api import create_new_run -from core.models import Credential, Destination, OAuthApiKeys, Source, StorageCredentials, Sync, Workspace + +from core.models import (Credential, Destination, OAuthApiKeys, Source, + StorageCredentials, Sync, Workspace) +from core.routes.workspace_api import create_new_run + logger = logging.getLogger(__name__) SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] diff --git a/valmi_app_backend/urls.py b/valmi_app_backend/urls.py index 9ad1927..ede55a1 100644 --- a/valmi_app_backend/urls.py +++ b/valmi_app_backend/urls.py @@ -8,134 +8,15 @@ import logging -from decouple import config from django.contrib import admin from django.urls import path -from ninja import NinjaAPI -from ninja.security import HttpBearer -from ninja.security import HttpBasicAuth - -from core.api import router as public_api_router -from core.stream_api import router as stream_api_router -from core.prompt_api import router as prompt_api_router -from core.package_api import router as package_api_router -from core.oauth_api import router as oauth_api_router -from core.explore_api import router as explore_api_router -from core.social_auth import router as social_api_router - -from core.api import get_workspaces -from core.engine_api import router as superuser_api_router +from core.routes.api import api from core.urls import core_urlpatterns -from valmi_app_backend.utils import BearerAuthentication - -from rest_framework.authentication import BasicAuthentication -from ninja.compatibility import get_headers -from django.conf import settings -from django.http import HttpRequest -from typing import Any, Optional logger = logging.getLogger(__name__) -class BasicAuth(HttpBasicAuth): - def authenticate(self, request, username, password): - try: - user_auth_tuple = BasicAuthentication().authenticate(request) - except Exception: - user_auth_tuple = None - if user_auth_tuple is not None: - (user, token) = user_auth_tuple - request.user = user - # Basic Auth allowed only for superuser. - if user.is_superuser: - return user - return None - return None - - -class AuthBearer(HttpBearer): - openapi_scheme: str = "bearer" - - def __call__(self, request: HttpRequest) -> Optional[Any]: - return self.authenticate(request) - - def has_permission_for(self, user, workspace_id): - for workspace in get_workspaces(user): - logger.debug("checking workspace %s", workspace.id) - if str(workspace.id) == workspace_id: - return True - return False - - def authenticate(self, request): - ''' - logger.debug("enabled " +config('PUBLIC_SYNC_ENABLED')) - logger.debug("pub " +config('PUBLIC_WORKSPACE')) - logger.debug("sync " +config('PUBLIC_SYNC')) - logger.debug("authtoken " +config('PUBLIC_AUTH_TOKEN')) - logger.debug("path "+ request.path) - logger.debug("hardcoded "+ f'/api/v1/workspaces/{config("PUBLIC_WORKSPACE")}/syncs/{config("PUBLIC_SYNC")}/runs/') - ''' - if (config('PUBLIC_SYNC_ENABLED', default=False) and - request.path == f'/api/v1/workspaces/{config("PUBLIC_WORKSPACE")}/syncs/{config("PUBLIC_SYNC")}/runs/'): - return config('PUBLIC_AUTH_TOKEN') - - headers = get_headers(request) - auth_value = headers.get(self.header) - if not auth_value: - return None - parts = auth_value.split(" ") - - if parts[0].lower() != self.openapi_scheme: - if settings.DEBUG: - logger.error(f"Unexpected auth - '{auth_value}'") - return None - token = " ".join(parts[1:]) - - try: - user_auth_tuple = BearerAuthentication().authenticate(request) - except Exception: - user_auth_tuple = None - if user_auth_tuple is not None: - (user, token) = user_auth_tuple # here come your user object - request.user = user - # get Workspace Id. - arr = request.get_full_path().split("/") - for i, el in enumerate(arr): - if el == "workspaces" and len(arr) > i + 1: - workspace_id = arr[i + 1] - logger.debug(workspace_id) - if not self.has_permission_for(user, workspace_id): - return None - break - return token - return None - - -api = NinjaAPI( - version="1.0", - csrf=False, - title="Valmi App Backend API", - description="App Backend API Serves the Valmi App Frontend", - urls_namespace="public_api", -) - - -api.add_router("/auth/social", social_api_router) -if config("AUTHENTICATION", default=True, cast=bool): - api.add_router("v1/superuser/", superuser_api_router, auth=[BasicAuth()]) - api.add_router("v1/streams/", stream_api_router, auth=[AuthBearer(), BasicAuth()]) - api.add_router("v1/prompts/", prompt_api_router, auth=[AuthBearer(), BasicAuth()]) - api.add_router("v1/packages/", package_api_router, auth=[AuthBearer(), BasicAuth()]) - api.add_router("v1/explores/", explore_api_router, auth=[AuthBearer(), BasicAuth()]) - api.add_router("v1/oauth/", oauth_api_router, auth=[AuthBearer(), BasicAuth()]) - api.add_router("v1/", public_api_router, auth=[AuthBearer(), BasicAuth()]) - -else: - api.add_router("v1/superuser/", superuser_api_router) - api.add_router("v1/streams/", stream_api_router) - api.add_router("v1/oauth/", oauth_api_router) - api.add_router("v1/", public_api_router) urlpatterns = [ path("admin/", admin.site.urls), From 3e7e0de6c80a2c25ae8d4142f485c1ec8e33580d Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 21 May 2024 17:36:03 +0530 Subject: [PATCH 087/159] feat: resolved merge conflicts --- core/routes/api.py | 21 ++++++++++++--------- core/routes/connector_api.py | 9 +++++---- core/routes/explore_api.py | 15 +++++++-------- core/routes/oauth_api.py | 2 +- core/routes/prompt_api.py | 10 ++++++---- core/routes/stream_api.py | 4 +--- core/services/explore_service.py | 2 +- 7 files changed, 33 insertions(+), 30 deletions(-) diff --git a/core/routes/api.py b/core/routes/api.py index 88558e8..366f7cf 100644 --- a/core/routes/api.py +++ b/core/routes/api.py @@ -6,6 +6,7 @@ """ +from core.schemas.schemas import UserSchemaOut import logging from typing import Any, Optional @@ -28,14 +29,15 @@ from core.routes.workspace_api import router as workspace_api_router from valmi_app_backend.utils import BearerAuthentication -from ..models import User +from core.models import User # Get an instance of a logger logger = logging.getLogger(__name__) -from core.schemas import UserSchemaOut auth_enabled = config("AUTHENTICATION", default=True, cast=bool) + + class BasicAuth(HttpBasicAuth): def authenticate(self, request, username, password): if auth_enabled == False: @@ -113,6 +115,7 @@ def authenticate(self, request): return token return None + def get_workspaces(user): queryset = User.objects.prefetch_related("organizations").get(id=user.id) for organization in queryset.organizations.all(): @@ -121,6 +124,8 @@ def get_workspaces(user): router = Router() + + @router.get("/spaces/", response=UserSchemaOut) def list_spaces(request): user_id = request.user.id @@ -129,8 +134,6 @@ def list_spaces(request): return queryset - - api = NinjaAPI( version="1.0", csrf=False, @@ -141,12 +144,12 @@ def list_spaces(request): api.add_router("/auth/social", social_api_router) -api.add_router("v1/", router, auth=[AuthBearer(),BasicAuth()]) +api.add_router("v1/", router, auth=[AuthBearer(), BasicAuth()]) router.add_router("superuser/", superuser_api_router, auth=[BasicAuth()]) -router.add_router("streams/",stream_api_router, tags = ["streams"]) +router.add_router("streams/", stream_api_router, tags=["streams"]) router.add_router("oauth/", oauth_api_router, tags=["oauth"]) router.add_router("connectors", connector_api_router, tags=["connectors"]) -router.add_router("packages", package_api_router, tags = ["packages"]) +router.add_router("packages", package_api_router, tags=["packages"]) router.add_router("", workspace_api_router, tags=["workspaces"]) -router.add_router("", prompt_api_router, tags = ["prompts"]) -router.add_router("", explore_api_router, tags = ["explores"]) +router.add_router("", prompt_api_router, tags=["prompts"]) +router.add_router("", explore_api_router, tags=["explores"]) diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py index ad43294..caf67c8 100644 --- a/core/routes/connector_api.py +++ b/core/routes/connector_api.py @@ -3,15 +3,16 @@ from ninja import Router -from core.schemas import ConnectorSchema, DetailSchema +from core.schemas.schemas import ConnectorSchema, DetailSchema -from ..models import Connector, OAuthApiKeys, Workspace +from core.models import Connector, OAuthApiKeys, Workspace router = Router() # Get an instance of a logger logger = logging.getLogger(__name__) + @router.get("/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) def get_connectors(request): # check for admin permissions @@ -37,7 +38,7 @@ def get_connectors(request): @router.get("{workspace_id}/configured", response={200: Dict[str, List[ConnectorSchema]], - 400: DetailSchema}) + 400: DetailSchema}) def get_connectors_configured(request, workspace_id): try: @@ -104,4 +105,4 @@ def get_connectors_not_configured(request, workspace_id): except Exception: logger.exception("connector listing error") - return (400, {"detail": "The list of connectors cannot be fetched."}) \ No newline at end of file + return (400, {"detail": "The list of connectors cannot be fetched."}) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 46702f8..f6967fb 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -12,9 +12,7 @@ from ninja import Router from pydantic import Json -from core.models import Account, Explore, Prompt, StorageCredentials, Workspace -from core.schemas import (DetailSchema, ExploreSchema, ExploreSchemaIn, - SyncStartStopSchemaIn) +from core.models import Account, Explore, Prompt, Workspace from core.services.explore_service import ExploreService logger = logging.getLogger(__name__) @@ -22,8 +20,9 @@ router = Router() ACTIVATION_URL = config("ACTIVATION_SERVER") + @router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchema], 400: DetailSchema}) -def get_explores(request,workspace_id): +def get_explores(request, workspace_id): try: logger.debug("listing explores") workspace = Workspace.objects.get(id=workspace_id) @@ -33,8 +32,8 @@ def get_explores(request,workspace_id): return (400, {"detail": "The list of explores cannot be fetched."}) -@router.post("/workspaces/{workspace_id}/explores/create",response={200: ExploreSchema, 400: DetailSchema}) -def create_explore(request, workspace_id,payload: ExploreSchemaIn): +@router.post("/workspaces/{workspace_id}/explores/create", response={200: ExploreSchema, 400: DetailSchema}) +def create_explore(request, workspace_id, payload: ExploreSchemaIn): data = payload.dict() try: data["id"] = uuid.uuid4() @@ -78,7 +77,7 @@ def create_explore(request, workspace_id,payload: ExploreSchemaIn): @router.get("/workspaces/{workspace_id}/explores/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_explores(request,workspace_id,explore_id): +def get_explores(request, workspace_id, explore_id): try: logger.debug("listing explores") return Explore.objects.get(id=explore_id) @@ -88,7 +87,7 @@ def get_explores(request,workspace_id,explore_id): @router.get("/workspaces/{workspace_id}/explores/{explore_id}/status", response={200: Json, 400: DetailSchema}) -def get_explore_status(request,workspace_id,explore_id): +def get_explore_status(request, workspace_id, explore_id): try: logger.debug("getting_explore_status") explore = Explore.objects.get(id=explore_id) diff --git a/core/routes/oauth_api.py b/core/routes/oauth_api.py index 0614204..c986ec0 100644 --- a/core/routes/oauth_api.py +++ b/core/routes/oauth_api.py @@ -16,7 +16,7 @@ from ninja import Router from pydantic import Json -from ..models import Connector, Workspace, OAuthApiKeys +from core.models import Connector, Workspace, OAuthApiKeys from core.schemas.schemas import ( DetailSchema, diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index c05d1e7..2a1bc76 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -8,15 +8,16 @@ from pydantic import Json from core.models import Credential, Prompt, Source, StorageCredentials from core.schemas.prompt import PromptPreviewSchemaIn -from core.schemas.schemas import DetailSchema, PromptByIdSchema, PromptSchemaOut +from core.schemas.schemas import DetailSchema from core.services.prompts import PromptService -from core.models import Prompt -from core.schemas import DetailSchema, PromptSchema +from core.models import Prompt, StorageCredentials +from core.schemas.schemas import DetailSchema, PromptSchema logger = logging.getLogger(__name__) router = Router() + @router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchema], 400: DetailSchema}) def get_prompts(request): try: @@ -34,8 +35,9 @@ def get_prompts(request): logger.exception("prompts listing error:" + err) return (400, {"detail": "The list of prompts cannot be fetched."}) + @router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) -def get_prompts(request,prompt_id): +def get_prompts(request, workspace_id, prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) diff --git a/core/routes/stream_api.py b/core/routes/stream_api.py index ff0247b..1a4f182 100644 --- a/core/routes/stream_api.py +++ b/core/routes/stream_api.py @@ -13,9 +13,8 @@ from ninja import Router from pydantic import Json from decouple import config -from core.api import SHORT_TIMEOUT from core.schemas.schemas import GenericJsonSchema -from .models import ValmiUserIDJitsuApiToken +from core.models import ValmiUserIDJitsuApiToken from typing import Optional import requests @@ -24,7 +23,6 @@ from pydantic import Json from core.routes.api_config import SHORT_TIMEOUT -from core.schemas import GenericJsonSchema from ..models import ValmiUserIDJitsuApiToken diff --git a/core/services/explore_service.py b/core/services/explore_service.py index 4f64904..c657676 100644 --- a/core/services/explore_service.py +++ b/core/services/explore_service.py @@ -11,7 +11,7 @@ from os.path import dirname, join from decouple import config import requests -from core.api import create_new_run +from core.routes.workspace_api import create_new_run from core.models import Credential, Destination, OAuthApiKeys, Prompt, Source, StorageCredentials, Sync, Workspace from core.schemas.prompt import Filter, TimeWindow from core.services.prompts import PromptService From d79be8bd271479a5b8c842d693dea81f76f4e632 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 21 May 2024 17:52:04 +0530 Subject: [PATCH 088/159] feat: used asyciio instead of time.sleep in explore creation --- core/routes/explore_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index f6967fb..e0330cf 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import json @@ -67,7 +68,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["spreadsheet_url"] = spreadsheet_url explore = Explore.objects.create(**data) # create run - ExploreService.wait_for_run(5) + asyncio.run(ExploreService.wait_for_run(5)) payload = SyncStartStopSchemaIn(full_refresh=True) ExploreService.create_run(request, workspace_id, sync.id, payload) return explore From dfa04a1217fcb8c545c7da6114a4bcc7a825c628 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 10:53:03 +0530 Subject: [PATCH 089/159] feat: renamed explore_service to explore in services folder --- core/routes/explore_api.py | 2 +- core/routes/prompt_api.py | 17 +++++++---------- .../services/{explore_service.py => explore.py} | 0 3 files changed, 8 insertions(+), 11 deletions(-) rename core/services/{explore_service.py => explore.py} (100%) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index e0330cf..52cb7c7 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -14,7 +14,7 @@ from pydantic import Json from core.models import Account, Explore, Prompt, Workspace -from core.services.explore_service import ExploreService +from core.services.explore import ExploreService logger = logging.getLogger(__name__) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 2a1bc76..631b754 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -8,36 +8,33 @@ from pydantic import Json from core.models import Credential, Prompt, Source, StorageCredentials from core.schemas.prompt import PromptPreviewSchemaIn -from core.schemas.schemas import DetailSchema +from core.schemas.schemas import DetailSchema, PromptByIdSchema from core.services.prompts import PromptService from core.models import Prompt, StorageCredentials -from core.schemas.schemas import DetailSchema, PromptSchema +from core.schemas.schemas import DetailSchema, PromptSchemaOut logger = logging.getLogger(__name__) router = Router() -@router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchema], 400: DetailSchema}) -def get_prompts(request): +@router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchemaOut], 400: DetailSchema}) +def get_prompts(request, workspace_id): try: prompts = list(Prompt.objects.all().values()) connector_ids = list(Credential.objects.values('connector_id').distinct()) connector_types = [connector['connector_id'] for connector in connector_ids] for prompt in prompts: prompt["id"] = str(prompt["id"]) - if prompt["type"] in connector_types: - prompt["enabled"] = True - else: - prompt["enabled"] = False + prompt["enabled"] = bool(PromptService.is_prompt_enabled(workspace_id, prompt)) return prompts except Exception as err: logger.exception("prompts listing error:" + err) return (400, {"detail": "The list of prompts cannot be fetched."}) -@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptSchema, 400: DetailSchema}) -def get_prompts(request, workspace_id, prompt_id): +@router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptByIdSchema, 400: DetailSchema}) +def get_prompt(request, workspace_id, prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) diff --git a/core/services/explore_service.py b/core/services/explore.py similarity index 100% rename from core/services/explore_service.py rename to core/services/explore.py From 5a9ff8681c94ae115c21431650a922e660752d0d Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 12:05:35 +0530 Subject: [PATCH 090/159] feat: schedule for shopify creation handled from backend --- core/schemas/schemas.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 97cad6d..1c890be 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -11,8 +11,7 @@ from django.contrib.auth.models import User from ninja import Field, ModelSchema, Schema from pydantic import UUID4 -from core.models import Account, Connector, Credential, Destination, Explore, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys -from core.schemas.prompt import Filter, TimeWindow +from core.models import Account, Connector, Credential, Destination, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys def camel_to_snake(s): @@ -87,10 +86,6 @@ class PromptSchemaOut(Schema): enabled: bool -class ExploreStatusIn(Schema): - sync_id: str - - class ConnectorConfigSchemaIn(Schema): config: Dict @@ -120,19 +115,6 @@ class ConnectionSchemaIn(Schema): source_connector_config: Dict -class ExploreSchemaIn(Schema): - name: str - account: Dict = None - prompt_id: str - schema_id: str - time_window: TimeWindow - filters: list[Filter] - - -class ExplorePreviewDataIn(Schema): - prompt_id: str - - class CredentialSchemaUpdateIn(Schema): id: UUID4 connector_type: str @@ -162,15 +144,6 @@ class Config(CamelSchemaConfig): account: AccountSchema = Field(None, alias="account") -class ExploreSchema(ModelSchema): - class Config(CamelSchemaConfig): - model = Explore - model_fields = ["ready", "name", "spreadsheet_url", "account", "id"] - account: AccountSchema = Field(None, alias="account") - prompt: PromptSchema = Field(None, alias="prompt") - workspace: WorkspaceSchema = Field(None, alias="workspace") - - class BaseSchemaIn(Schema): workspace_id: UUID4 @@ -215,7 +188,7 @@ class SyncSchemaInWithSourcePayload(Schema): name: str source: Dict account: Optional[Dict] - schedule: Dict + schedule: Optional[Dict] ui_state: Optional[Dict] @@ -305,7 +278,3 @@ class SocialUser(Schema): class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser - - -class ExploreStatusSchemaIn(Schema): - sync_id: str From 2deacd2422fcb4eecbb14ed188b2981206f040f3 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 12:06:17 +0530 Subject: [PATCH 091/159] feat: Schedule for shopify creation is handled from backend --- core/schemas/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 1c890be..8420e2e 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -197,7 +197,7 @@ class SyncSchemaUpdateIn(Schema): name: str source_id: UUID4 destination_id: UUID4 - schedule: Dict + schedule: Optional[Dict] ui_state: Optional[Dict] From eed0f0b9ca314e1f21f02293047380170fb731db Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 12:28:17 +0530 Subject: [PATCH 092/159] feat: resolved get prompts --- core/routes/prompt_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 631b754..4a599bd 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -26,7 +26,10 @@ def get_prompts(request, workspace_id): connector_types = [connector['connector_id'] for connector in connector_ids] for prompt in prompts: prompt["id"] = str(prompt["id"]) - prompt["enabled"] = bool(PromptService.is_prompt_enabled(workspace_id, prompt)) + if prompt["type"] in connector_types: + prompt["enabled"] = True + else: + prompt["enabled"] = False return prompts except Exception as err: logger.exception("prompts listing error:" + err) From b2df7e7ba8cc4bc34fcdb35467d33d535a43a0ff Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 12:34:17 +0530 Subject: [PATCH 093/159] feat: created new file for explore schemas --- core/routes/explore_api.py | 14 ++++++++++---- core/routes/workspace_api.py | 22 +++++++++++++++++---- core/schemas/explore.py | 37 ++++++++++++++++++++++++++++++++++++ core/services/explore.py | 6 ++++++ 4 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 core/schemas/explore.py diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 52cb7c7..11e4ae2 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -9,7 +9,8 @@ from pydantic import Json import requests from core.models import Account, Explore, Prompt, Workspace -from core.schemas.schemas import DetailSchema, ExploreSchema, ExploreSchemaIn, SyncStartStopSchemaIn +from core.schemas.schemas import DetailSchema, SyncStartStopSchemaIn +from core.schemas.explore import ExploreSchema, ExploreSchemaIn, ExploreSchemaOut from ninja import Router from pydantic import Json @@ -22,12 +23,17 @@ ACTIVATION_URL = config("ACTIVATION_SERVER") -@router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchema], 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchemaOut], 400: DetailSchema}) def get_explores(request, workspace_id): try: logger.debug("listing explores") workspace = Workspace.objects.get(id=workspace_id) - return Explore.objects.filter(workspace=workspace).order_by('created_at') + explores = Explore.objects.filter(workspace=workspace).order_by('created_at') + for explore in explores: + explore.prompt_id = explore.prompt.id + explore.workspace_id = explore.workspace.id + explore.last_successful_time = ExploreService.get_last_sync_successful_time(request, explore.sync.id) + return explores except Exception: logger.exception("explores listing error") return (400, {"detail": "The list of explores cannot be fetched."}) @@ -69,7 +75,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): explore = Explore.objects.create(**data) # create run asyncio.run(ExploreService.wait_for_run(5)) - payload = SyncStartStopSchemaIn(full_refresh=True) + payload = SyncStartStopSchemaIn(full_refresh=False) ExploreService.create_run(request, workspace_id, sync.id, payload) return explore except Exception as e: diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 80aef97..36d42b3 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -323,6 +323,14 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): @router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 400: DetailSchema}) def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data = payload.dict() + key_to_check = "schedule" + if data["schedule"] is None: + schedule = {"run_interval": 3600000} + data["schedule"] = schedule + logger.debug("---------------------------------------") + logger.debug(data["schedule"]["run_interval"]) + return + logger.debug(data) source_config = data["source"]["config"] catalog = data["source"]["catalog"] for stream in catalog["streams"]: @@ -354,11 +362,17 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data["source"] = Source.objects.get(id=source.id) data["destination"] = Destination.objects.get(id=destination.id) del data["account"] - schedule = {} - if len(data["schedule"]) == 0: - schedule["run_interval"] = 3600000 - data["schedule"] = schedule + key_to_check = "schedule" + value = data.get(key_to_check) + if "schedule" not in data: + schedule = {"run_interval": 3600000} + data[key_to_check] = schedule + logger.debug(data["schedule"]) data["workspace"] = Workspace.objects.get(id=workspace_id) + key_to_check = "ui_state" + if key_to_check not in data: + ui_state = {} + data[key_to_check] = ui_state data["id"] = uuid.uuid4() logger.debug(data) sync = Sync.objects.create(**data) diff --git a/core/schemas/explore.py b/core/schemas/explore.py new file mode 100644 index 0000000..9241596 --- /dev/null +++ b/core/schemas/explore.py @@ -0,0 +1,37 @@ +from typing import Dict +from ninja import Field, ModelSchema, Schema +from core.models import Explore +from core.schemas.prompt import Filter, TimeWindow +from core.schemas.schemas import AccountSchema, CamelSchemaConfig, PromptSchema, WorkspaceSchema + + +class ExploreSchemaOut(Schema): + ready:bool + name:str + spreadsheet_url:str + id:str + prompt_id:str + workspace_id:str + last_successful_time:str + created_at:str + +class ExploreSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Explore + model_fields = ["ready", "name", "spreadsheet_url", "account", "id"] + account: AccountSchema = Field(None, alias="account") + prompt: PromptSchema = Field(None, alias="prompt") + workspace: WorkspaceSchema = Field(None, alias="workspace") + + +class ExploreStatusIn(Schema): + sync_id: str + + +class ExploreSchemaIn(Schema): + name: str + account: Dict = None + prompt_id: str + schema_id: str + time_window: TimeWindow + filters: list[Filter] diff --git a/core/services/explore.py b/core/services/explore.py index c657676..817a925 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -211,3 +211,9 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object except Exception as e: logger.exception(f"Error creating run: {e}") raise Exception("unable to create run") + + @staticmethod + def get_last_sync_successful_time(request: object, sync_id: object) -> str: + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") + status = response.text + print(status) From c0a4e4c5021e678c03f90a7ec64bf0bc808fdc60 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 22 May 2024 13:36:10 +0530 Subject: [PATCH 094/159] feat: handled schedule for shopify sync from backend --- core/routes/workspace_api.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 36d42b3..20a1975 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -323,15 +323,6 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): @router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 400: DetailSchema}) def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data = payload.dict() - key_to_check = "schedule" - if data["schedule"] is None: - schedule = {"run_interval": 3600000} - data["schedule"] = schedule - logger.debug("---------------------------------------") - logger.debug(data["schedule"]["run_interval"]) - return - logger.debug(data) - source_config = data["source"]["config"] catalog = data["source"]["catalog"] for stream in catalog["streams"]: primary_key = [["id"]] @@ -339,8 +330,8 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): stream["destination_sync_mode"] = "append_dedup" # creating source credential source_credential_payload = CredentialSchemaIn( - name=source_config["name"], account=data["account"], connector_type=source_config["source_connector_type"], - connector_config=source_config["source_connector_config"]) + name="shopify", account=data["account"], connector_type=data["source"]["type"], + connector_config=data["source"]["config"]) source_credential = create_credential(request, workspace_id, source_credential_payload) # creating source source_payload = SourceSchemaIn( @@ -362,17 +353,13 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data["source"] = Source.objects.get(id=source.id) data["destination"] = Destination.objects.get(id=destination.id) del data["account"] - key_to_check = "schedule" - value = data.get(key_to_check) - if "schedule" not in data: + if data["schedule"] is None: schedule = {"run_interval": 3600000} - data[key_to_check] = schedule - logger.debug(data["schedule"]) + data["schedule"] = schedule data["workspace"] = Workspace.objects.get(id=workspace_id) - key_to_check = "ui_state" - if key_to_check not in data: + if data["ui_state"] is None: ui_state = {} - data[key_to_check] = ui_state + data["ui_state"] = ui_state data["id"] = uuid.uuid4() logger.debug(data) sync = Sync.objects.create(**data) From 93161ca09af1a501caae5176e5e193046424222e Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 23 May 2024 16:16:33 +0530 Subject: [PATCH 095/159] feat: Update output schema of Explores to include sync_state,last_sync_result fields --- core/routes/explore_api.py | 25 ++++++++++++++++++++++--- core/routes/prompt_api.py | 9 +++------ core/schemas/explore.py | 23 ++++++++++++++--------- core/services/explore.py | 21 ++++++++++++++++++--- core/services/prompts.py | 4 +++- 5 files changed, 60 insertions(+), 22 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 11e4ae2..aecc541 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -30,9 +30,28 @@ def get_explores(request, workspace_id): workspace = Workspace.objects.get(id=workspace_id) explores = Explore.objects.filter(workspace=workspace).order_by('created_at') for explore in explores: - explore.prompt_id = explore.prompt.id - explore.workspace_id = explore.workspace.id - explore.last_successful_time = ExploreService.get_last_sync_successful_time(request, explore.sync.id) + explore.prompt_id = str(explore.prompt.id) + explore.workspace_id = str(explore.workspace.id) + explore.id = str(explore.id) + explore_sync_status = ExploreService.is_explore_running(explore.sync.id) + if explore_sync_status.get('enabled') == False: + explore.enabled = False + explore.sync_state = 'IDLE' + explore.last_sync_result = 'UNKNOWN' + explore.last_sync_created_at = "" + explore.last_sync_succeeded_at = "" + explore.sync_id = "" + continue + explore.enabled = True + explore.sync_id = str(explore.sync.id) + explore.last_sync_succeeded_at = ExploreService.get_last_sync_successful_time(explore.sync.id) + if explore_sync_status.get('is_running') == True: + explore.sync_state = 'RUNNING' + explore.last_sync_result = 'UNKNOWN' + else: + explore.last_sync_result = explore_sync_status.get('status') + explore.sync_state = 'IDLE' + explore.last_sync_created_at = explore_sync_status.get('created_at') return explores except Exception: logger.exception("explores listing error") diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 4a599bd..ec53b3e 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -26,10 +26,7 @@ def get_prompts(request, workspace_id): connector_types = [connector['connector_id'] for connector in connector_ids] for prompt in prompts: prompt["id"] = str(prompt["id"]) - if prompt["type"] in connector_types: - prompt["enabled"] = True - else: - prompt["enabled"] = False + prompt["enabled"] = PromptService.is_enabled(workspace_id, prompt) return prompts except Exception as err: logger.exception("prompts listing error:" + err) @@ -41,7 +38,7 @@ def get_prompt(request, workspace_id, prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) - if not PromptService.is_prompt_enabled(workspace_id, prompt): + if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} credential_info = Source.objects.filter( @@ -83,7 +80,7 @@ def custom_serializer(obj): def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSchemaIn): try: prompt = Prompt.objects.get(id=prompt_id) - if not PromptService.is_prompt_enabled(workspace_id, prompt): + if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) diff --git a/core/schemas/explore.py b/core/schemas/explore.py index 9241596..6262ca7 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Optional from ninja import Field, ModelSchema, Schema from core.models import Explore from core.schemas.prompt import Filter, TimeWindow @@ -6,14 +6,19 @@ class ExploreSchemaOut(Schema): - ready:bool - name:str - spreadsheet_url:str - id:str - prompt_id:str - workspace_id:str - last_successful_time:str - created_at:str + ready: bool + name: str + spreadsheet_url: str + id: str + prompt_id: str + enabled: bool + workspace_id: str + last_sync_succeeded_at: str + last_sync_created_at: str + last_sync_result: str + sync_state: str + sync_id: str + class ExploreSchema(ModelSchema): class Config(CamelSchemaConfig): diff --git a/core/services/explore.py b/core/services/explore.py index 817a925..a9b2e51 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -1,4 +1,5 @@ import asyncio +from datetime import datetime import json import logging import os @@ -213,7 +214,21 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object raise Exception("unable to create run") @staticmethod - def get_last_sync_successful_time(request: object, sync_id: object) -> str: + def get_last_sync_successful_time(sync_id: object) -> str: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") - status = response.text - print(status) + json_string = response.content.decode('utf-8') + dict_data = json.loads(json_string) + if dict_data.get('found') == True: + timestamp = datetime.strptime(dict_data.get('timestamp'), "%Y-%m-%dT%H:%M:%S.%f") + timestamp = timestamp.replace(microsecond=0) + return timestamp.isoformat() + return "" + + @staticmethod + def is_explore_running(sync_id: str) -> dict: + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") + json_string = response.content.decode('utf-8') + dict_data = json.loads(json_string) + logger.debug(dict_data) + dict_data["is_running"] = dict_data.get('status') == 'running' + return dict_data diff --git a/core/services/prompts.py b/core/services/prompts.py index d8cfbac..49a91f0 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -38,7 +38,9 @@ def build(table, timeWindow: TimeWindow, filters: list[Filter]) -> str: return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) @staticmethod - def is_prompt_enabled(workspace_id: str, prompt: object) -> bool: + def is_enabled(workspace_id: str, prompt: object) -> bool: connector_ids = list(Credential.objects.filter(workspace_id=workspace_id).values('connector_id').distinct()) connector_types = [connector['connector_id'] for connector in connector_ids] + if isinstance(prompt, dict): + return prompt["type"] in connector_types return prompt.type in connector_types From 230d5dd498502308afd6dd8aeba9f20a828b1b58 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 23 May 2024 16:25:20 +0530 Subject: [PATCH 096/159] feat: renamed timestamp to run_end_at in explore service --- core/services/explore.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index a9b2e51..d9d6d16 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -1,5 +1,4 @@ import asyncio -from datetime import datetime import json import logging import os @@ -213,16 +212,13 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object logger.exception(f"Error creating run: {e}") raise Exception("unable to create run") + @staticmethod @staticmethod def get_last_sync_successful_time(sync_id: object) -> str: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") json_string = response.content.decode('utf-8') dict_data = json.loads(json_string) - if dict_data.get('found') == True: - timestamp = datetime.strptime(dict_data.get('timestamp'), "%Y-%m-%dT%H:%M:%S.%f") - timestamp = timestamp.replace(microsecond=0) - return timestamp.isoformat() - return "" + return dict_data.get('run_end_at') if dict_data.get('found') == True else "" @staticmethod def is_explore_running(sync_id: str) -> dict: From 234fbc94efcdc6d21fa4827f2d53a509de80a061 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 23 May 2024 16:27:30 +0530 Subject: [PATCH 097/159] feat: renamed is_explore_running to explore_running in explore service --- core/routes/explore_api.py | 2 +- core/services/explore.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index aecc541..bbf757c 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -33,7 +33,7 @@ def get_explores(request, workspace_id): explore.prompt_id = str(explore.prompt.id) explore.workspace_id = str(explore.workspace.id) explore.id = str(explore.id) - explore_sync_status = ExploreService.is_explore_running(explore.sync.id) + explore_sync_status = ExploreService.is_running(explore.sync.id) if explore_sync_status.get('enabled') == False: explore.enabled = False explore.sync_state = 'IDLE' diff --git a/core/services/explore.py b/core/services/explore.py index d9d6d16..4897f5c 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -212,7 +212,6 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object logger.exception(f"Error creating run: {e}") raise Exception("unable to create run") - @staticmethod @staticmethod def get_last_sync_successful_time(sync_id: object) -> str: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") @@ -221,7 +220,7 @@ def get_last_sync_successful_time(sync_id: object) -> str: return dict_data.get('run_end_at') if dict_data.get('found') == True else "" @staticmethod - def is_explore_running(sync_id: str) -> dict: + def is_running(sync_id: str) -> dict: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") json_string = response.content.decode('utf-8') dict_data = json.loads(json_string) From 9534401b43cf4d004dccca12f18de2fede96985c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 23 May 2024 17:10:47 +0530 Subject: [PATCH 098/159] feat: renamed is_running to is_sync_created_or_running in explore service --- core/routes/explore_api.py | 3 ++- core/services/explore.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index bbf757c..2b7dbd9 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -33,8 +33,9 @@ def get_explores(request, workspace_id): explore.prompt_id = str(explore.prompt.id) explore.workspace_id = str(explore.workspace.id) explore.id = str(explore.id) - explore_sync_status = ExploreService.is_running(explore.sync.id) + explore_sync_status = ExploreService.is_sync_created_or_running(explore.sync.id) if explore_sync_status.get('enabled') == False: + explore.enabled = False explore.sync_state = 'IDLE' explore.last_sync_result = 'UNKNOWN' diff --git a/core/services/explore.py b/core/services/explore.py index 4897f5c..0c55d20 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -220,7 +220,7 @@ def get_last_sync_successful_time(sync_id: object) -> str: return dict_data.get('run_end_at') if dict_data.get('found') == True else "" @staticmethod - def is_running(sync_id: str) -> dict: + def is_sync_created_or_running(sync_id: str) -> dict: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") json_string = response.content.decode('utf-8') dict_data = json.loads(json_string) From c3e95ed09341d99784b9be042e7c80c405300b9c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 11:32:15 +0530 Subject: [PATCH 099/159] feat: added description for explores --- core/routes/explore_api.py | 10 +++++----- core/schemas/explore.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 2b7dbd9..96b4513 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -31,17 +31,17 @@ def get_explores(request, workspace_id): explores = Explore.objects.filter(workspace=workspace).order_by('created_at') for explore in explores: explore.prompt_id = str(explore.prompt.id) + explore.description = Prompt.objects.get(id=explore.prompt.id).description explore.workspace_id = str(explore.workspace.id) explore.id = str(explore.id) + explore.sync_id = str(explore.sync.id) explore_sync_status = ExploreService.is_sync_created_or_running(explore.sync.id) if explore_sync_status.get('enabled') == False: - explore.enabled = False explore.sync_state = 'IDLE' explore.last_sync_result = 'UNKNOWN' explore.last_sync_created_at = "" explore.last_sync_succeeded_at = "" - explore.sync_id = "" continue explore.enabled = True explore.sync_id = str(explore.sync.id) @@ -50,7 +50,7 @@ def get_explores(request, workspace_id): explore.sync_state = 'RUNNING' explore.last_sync_result = 'UNKNOWN' else: - explore.last_sync_result = explore_sync_status.get('status') + explore.last_sync_result = explore_sync_status.get('status').upper() explore.sync_state = 'IDLE' explore.last_sync_created_at = explore_sync_status.get('created_at') return explores @@ -78,7 +78,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): source = ExploreService.create_source( data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) # create destination - spreadsheet_name = f"valmiio {prompt.name} sheet" + spreadsheet_name = f"valmi.io {prompt.name} sheet" destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) spreadsheet_url = destination_data[0] destination = destination_data[1] @@ -88,7 +88,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): del data["schema_id"] del data["filters"] del data["time_window"] - data["name"] = f"valmiio {prompt.name}" + # data["name"] = f"valmiio {prompt.name}" data["sync"] = sync data["ready"] = False data["spreadsheet_url"] = spreadsheet_url diff --git a/core/schemas/explore.py b/core/schemas/explore.py index 6262ca7..a594028 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Dict from ninja import Field, ModelSchema, Schema from core.models import Explore from core.schemas.prompt import Filter, TimeWindow @@ -12,6 +12,7 @@ class ExploreSchemaOut(Schema): id: str prompt_id: str enabled: bool + description: str workspace_id: str last_sync_succeeded_at: str last_sync_created_at: str From 556d47b3629d2ffc6fabd2da5669dac1fe96859b Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 11:42:07 +0530 Subject: [PATCH 100/159] feat: resolved bug spec generation using oauth endpoint --- core/{ => routes}/oauth_schema.json | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) rename core/{ => routes}/oauth_schema.json (88%) diff --git a/core/oauth_schema.json b/core/routes/oauth_schema.json similarity index 88% rename from core/oauth_schema.json rename to core/routes/oauth_schema.json index 549e187..05b25cb 100644 --- a/core/oauth_schema.json +++ b/core/routes/oauth_schema.json @@ -11,7 +11,8 @@ "AUTH_FACEBOOK_CLIENT_SECRET": { "type": "string", "description": "Facebook client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" } }, @@ -34,7 +35,8 @@ "AUTH_GOOGLE_CLIENT_SECRET": { "type": "string", "description": "Google client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" } }, "required": [ @@ -56,13 +58,15 @@ "AUTH_GOOGLE_CLIENT_SECRET": { "type": "string", "description": "Google client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" }, "AUTH_GOOGLE_DEVELOPER_TOKEN": { "type": "string", "description": "Google developer token", - "title":"Developer Token" + "title":"Developer Token", + "format":"password" } }, "required": [ @@ -84,7 +88,8 @@ "AUTH_SLACK_CLIENT_SECRET": { "type": "string", "description": "Slack client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" } }, @@ -107,7 +112,8 @@ "AUTH_HUBSPOT_CLIENT_SECRET": { "type": "string", "description": "Hubspot client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" } }, @@ -129,7 +135,8 @@ "AUTH_SHOPIFY_CLIENT_SECRET": { "type": "string", "description": "Shopify client secret", - "title":"Client Secret" + "title":"Client Secret", + "format":"password" } }, From d498d9190e8455140a80865e17f536e3e35d280c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 13:20:31 +0530 Subject: [PATCH 101/159] feat: restrict prompt data preview until Shopify to DB sync is successful --- core/routes/prompt_api.py | 2 ++ core/services/explore.py | 2 +- core/services/prompts.py | 67 ++++++++++++++++++++++++++------------- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index ec53b3e..c12bf9c 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -83,6 +83,8 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} + if not PromptService.is_sync_finished(prompt_req.schema_id): + return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] table_name = f'{schema_name}.{prompt.table}' diff --git a/core/services/explore.py b/core/services/explore.py index 0c55d20..f352381 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -213,7 +213,7 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object raise Exception("unable to create run") @staticmethod - def get_last_sync_successful_time(sync_id: object) -> str: + def get_last_sync_successful_time(sync_id: str) -> str: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") json_string = response.content.decode('utf-8') dict_data = json.loads(json_string) diff --git a/core/services/prompts.py b/core/services/prompts.py index 49a91f0..2ad27cb 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,12 +1,17 @@ +import json import logging from pathlib import Path -from core.models import Credential +import requests + + +from core.models import Credential, SourceAccessInfo, Sync from liquid import Environment, FileSystemLoader, Mode, StrictUndefined from core.schemas.prompt import TimeWindow, Filter - +from decouple import config logger = logging.getLogger(__name__) +ACTIVATION_URL = config("ACTIVATION_SERVER") class PromptService(): @@ -16,26 +21,30 @@ def getTemplateFile(cls): @staticmethod def build(table, timeWindow: TimeWindow, filters: list[Filter]) -> str: - if isinstance(timeWindow, TimeWindow): - timeWindowDict = timeWindow.dict() - else: - timeWindowDict = timeWindow - if isinstance(filters, Filter): - filterList = [filter.__dict__ for filter in filters] - else: - filterList = filters - file_name = PromptService.getTemplateFile() - template_parent_path = Path(__file__).parent.absolute() - env = Environment( - tolerance=Mode.STRICT, - undefined=StrictUndefined, - loader=FileSystemLoader( - f"{str(template_parent_path)}/prompt_templates" - ), - ) - template = env.get_template(file_name) - filters = list(filters) - return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) + try: + if isinstance(timeWindow, TimeWindow): + timeWindowDict = timeWindow.dict() + else: + timeWindowDict = timeWindow + if isinstance(filters, Filter): + filterList = [filter.__dict__ for filter in filters] + else: + filterList = filters + file_name = PromptService.getTemplateFile() + template_parent_path = Path(__file__).parent.absolute() + env = Environment( + tolerance=Mode.STRICT, + undefined=StrictUndefined, + loader=FileSystemLoader( + f"{str(template_parent_path)}/prompt_templates" + ), + ) + template = env.get_template(file_name) + filters = list(filters) + return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) + except Exception as e: + logger.exception(e) + raise e @staticmethod def is_enabled(workspace_id: str, prompt: object) -> bool: @@ -44,3 +53,17 @@ def is_enabled(workspace_id: str, prompt: object) -> bool: if isinstance(prompt, dict): return prompt["type"] in connector_types return prompt.type in connector_types + + @staticmethod + def is_sync_finished(schema_id: str) -> bool: + try: + source_access_info = SourceAccessInfo.objects.get(storage_credentials_id=schema_id) + sync = Sync.objects.get(source_id=source_access_info.source.id) + sync_id = sync.id + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") + json_string = response.content.decode('utf-8') + dict_data = json.loads(json_string) + return dict_data["found"] == True + except Exception as e: + logger.exception(e) + raise e From 22f59550a754ec03571bad1aad4299eeca89bc2f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 13:31:17 +0530 Subject: [PATCH 102/159] feat: remove status endpoint for explores --- core/routes/explore_api.py | 42 -------------------------------------- 1 file changed, 42 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 96b4513..80dddff 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -1,18 +1,12 @@ import asyncio -import json import logging -import json from typing import List import uuid from decouple import config -import json -from pydantic import Json -import requests from core.models import Account, Explore, Prompt, Workspace from core.schemas.schemas import DetailSchema, SyncStartStopSchemaIn from core.schemas.explore import ExploreSchema, ExploreSchemaIn, ExploreSchemaOut from ninja import Router -from pydantic import Json from core.models import Account, Explore, Prompt, Workspace from core.services.explore import ExploreService @@ -111,39 +105,3 @@ def get_explores(request, workspace_id, explore_id): except Exception: logger.exception("explore listing error") return (400, {"detail": "The explore cannot be fetched."}) - - -@router.get("/workspaces/{workspace_id}/explores/{explore_id}/status", response={200: Json, 400: DetailSchema}) -def get_explore_status(request, workspace_id, explore_id): - try: - logger.debug("getting_explore_status") - explore = Explore.objects.get(id=explore_id) - response = {} - if explore.ready: - response["status"] = "success" - return json.dumps(response) - sync_id = explore.sync.id - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latestRunStatus") - status = response.text - print(status) - # if status == 'stopped': - # CODE for re running the sync from backend - # payload = SyncStartStopSchemaIn(full_refresh=True) - # response = create_new_run(request,workspace_id,sync_id,payload) - # print(response) - # return "sync got failed. Please re-try again" - if status == '"running"': - explore.ready = False - response["status"] = "running" - return json.dumps(response) - if status == '"failed"': - explore.ready = False - response["status"] = "failed" - return json.dumps(response) - explore.ready = True - explore.save() - response["status"] = "success" - return json.dumps(response) - except Exception: - logger.exception("get_explore_status error") - return (400, {"detail": "The explore cannot be fetched."}) From 3177746c28dfe136138eb9bd49a6277dc8fbf2a3 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 13:32:57 +0530 Subject: [PATCH 103/159] feat: remove status endpoint for explores --- core/routes/explore_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 80dddff..6b7da31 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -38,7 +38,6 @@ def get_explores(request, workspace_id): explore.last_sync_succeeded_at = "" continue explore.enabled = True - explore.sync_id = str(explore.sync.id) explore.last_sync_succeeded_at = ExploreService.get_last_sync_successful_time(explore.sync.id) if explore_sync_status.get('is_running') == True: explore.sync_state = 'RUNNING' From 4912491251793b66885ef1d921505bd93a5f530e Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 24 May 2024 13:34:39 +0530 Subject: [PATCH 104/159] feat: remove status endpoint for explores --- core/routes/explore_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 6b7da31..8905c91 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -38,7 +38,6 @@ def get_explores(request, workspace_id): explore.last_sync_succeeded_at = "" continue explore.enabled = True - explore.last_sync_succeeded_at = ExploreService.get_last_sync_successful_time(explore.sync.id) if explore_sync_status.get('is_running') == True: explore.sync_state = 'RUNNING' explore.last_sync_result = 'UNKNOWN' @@ -46,6 +45,7 @@ def get_explores(request, workspace_id): explore.last_sync_result = explore_sync_status.get('status').upper() explore.sync_state = 'IDLE' explore.last_sync_created_at = explore_sync_status.get('created_at') + explore.last_sync_succeeded_at = ExploreService.get_last_sync_successful_time(explore.sync.id) return explores except Exception: logger.exception("explores listing error") From f67abe96ff960b71497304c0eba9b4feb79458ed Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 27 May 2024 10:40:50 +0530 Subject: [PATCH 105/159] feat: Added exception handling in explore service --- core/services/explore.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index f352381..a43b4d5 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -214,16 +214,24 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object @staticmethod def get_last_sync_successful_time(sync_id: str) -> str: - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") - json_string = response.content.decode('utf-8') - dict_data = json.loads(json_string) - return dict_data.get('run_end_at') if dict_data.get('found') == True else "" + try: + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") + json_string = response.content.decode('utf-8') + dict_data = json.loads(json_string) + return dict_data.get('run_end_at') if dict_data.get('found') == True else "" + except Exception as e: + logger.exception(f"Error : {e}") + raise Exception("unable to query activation") @staticmethod def is_sync_created_or_running(sync_id: str) -> dict: - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") - json_string = response.content.decode('utf-8') - dict_data = json.loads(json_string) - logger.debug(dict_data) - dict_data["is_running"] = dict_data.get('status') == 'running' - return dict_data + try: + response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") + json_string = response.content.decode('utf-8') + dict_data = json.loads(json_string) + logger.debug(dict_data) + dict_data["is_running"] = dict_data.get('status') == 'running' + return dict_data + except Exception as e: + logger.exception(f"Error : {e}") + raise Exception("unable to query activation") From d487e928425551ad3a7231bf05549e766a1a96c4 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 28 May 2024 10:59:35 +0530 Subject: [PATCH 106/159] feat: Used schemas in prompt,explore services instead of dict --- core/routes/explore_api.py | 36 +++++++++------ core/routes/prompt_api.py | 16 +++++-- core/routes/workspace_api.py | 90 +++++++++++++++++++----------------- core/schemas/explore.py | 13 ++++-- core/schemas/prompt.py | 8 +++- core/services/explore.py | 21 ++------- core/services/prompts.py | 16 +++---- 7 files changed, 110 insertions(+), 90 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 8905c91..4504863 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -10,6 +10,7 @@ from core.models import Account, Explore, Prompt, Workspace from core.services.explore import ExploreService +from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -17,7 +18,7 @@ ACTIVATION_URL = config("ACTIVATION_SERVER") -@router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchemaOut], 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/explores", response={200: List[ExploreSchemaOut], 500: DetailSchema}) def get_explores(request, workspace_id): try: logger.debug("listing explores") @@ -29,27 +30,33 @@ def get_explores(request, workspace_id): explore.workspace_id = str(explore.workspace.id) explore.id = str(explore.id) explore.sync_id = str(explore.sync.id) - explore_sync_status = ExploreService.is_sync_created_or_running(explore.sync.id) - if explore_sync_status.get('enabled') == False: + latest_sync_info = ExploreService.get_latest_sync_info(explore.sync.id) + logger.debug(latest_sync_info) + # checking whether run is created for explore or not + if latest_sync_info.found == False: explore.enabled = False explore.sync_state = 'IDLE' explore.last_sync_result = 'UNKNOWN' - explore.last_sync_created_at = "" - explore.last_sync_succeeded_at = "" continue explore.enabled = True - if explore_sync_status.get('is_running') == True: + # checking the run status + if latest_sync_info.status == 'running': explore.sync_state = 'RUNNING' explore.last_sync_result = 'UNKNOWN' else: - explore.last_sync_result = explore_sync_status.get('status').upper() + explore.last_sync_result = latest_sync_info.status.upper() explore.sync_state = 'IDLE' - explore.last_sync_created_at = explore_sync_status.get('created_at') - explore.last_sync_succeeded_at = ExploreService.get_last_sync_successful_time(explore.sync.id) + explore.last_sync_created_at = latest_sync_info.created_at + # adding last successful sync info + last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) + if last_successful_sync_info.found == True: + explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at + else: + explore.last_sync_succeeded_at = "" return explores except Exception: logger.exception("explores listing error") - return (400, {"detail": "The list of explores cannot be fetched."}) + return (500, {"detail": "The list of explores cannot be fetched."}) @router.post("/workspaces/{workspace_id}/explores/create", response={200: ExploreSchema, 400: DetailSchema}) @@ -89,18 +96,19 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): # create run asyncio.run(ExploreService.wait_for_run(5)) payload = SyncStartStopSchemaIn(full_refresh=False) - ExploreService.create_run(request, workspace_id, sync.id, payload) + response = ExploreService.create_run(request, workspace_id, sync.id, payload) + logger.debug(response) return explore except Exception as e: logger.exception(e) return (400, {"detail": "The specific explore cannot be created."}) -@router.get("/workspaces/{workspace_id}/explores/{explore_id}", response={200: ExploreSchema, 400: DetailSchema}) -def get_explores(request, workspace_id, explore_id): +@router.get("/workspaces/{workspace_id}/explores/{explore_id}", response={200: ExploreSchema, 500: DetailSchema}) +def get_explore_by_id(request, workspace_id, explore_id): try: logger.debug("listing explores") return Explore.objects.get(id=explore_id) except Exception: logger.exception("explore listing error") - return (400, {"detail": "The explore cannot be fetched."}) + return (500, {"detail": "The explore cannot be fetched."}) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index c12bf9c..a21731b 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -6,7 +6,7 @@ from ninja import Router import psycopg2 from pydantic import Json -from core.models import Credential, Prompt, Source, StorageCredentials +from core.models import Credential, Prompt, Source, SourceAccessInfo, StorageCredentials, Sync from core.schemas.prompt import PromptPreviewSchemaIn from core.schemas.schemas import DetailSchema, PromptByIdSchema from core.services.prompts import PromptService @@ -18,7 +18,7 @@ router = Router() -@router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchemaOut], 400: DetailSchema}) +@router.get("/workspaces/{workspace_id}/prompts", response={200: List[PromptSchemaOut], 500: DetailSchema}) def get_prompts(request, workspace_id): try: prompts = list(Prompt.objects.all().values()) @@ -30,11 +30,11 @@ def get_prompts(request, workspace_id): return prompts except Exception as err: logger.exception("prompts listing error:" + err) - return (400, {"detail": "The list of prompts cannot be fetched."}) + return (500, {"detail": "The list of prompts cannot be fetched."}) @router.get("/workspaces/{workspace_id}/prompts/{prompt_id}", response={200: PromptByIdSchema, 400: DetailSchema}) -def get_prompt(request, workspace_id, prompt_id): +def get_prompt_by_id(request, workspace_id, prompt_id): try: logger.debug("listing prompts") prompt = Prompt.objects.get(id=prompt_id) @@ -80,10 +80,16 @@ def custom_serializer(obj): def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSchemaIn): try: prompt = Prompt.objects.get(id=prompt_id) + # checking wether prompt is enabled or not if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" return 400, {"detail": detail_message} - if not PromptService.is_sync_finished(prompt_req.schema_id): + source_access_info = SourceAccessInfo.objects.get(storage_credentials_id=prompt_req.schema_id) + sync = Sync.objects.get(source_id=source_access_info.source.id) + sync_id = sync.id + # checking wether sync has finished or not(from shopify to DB) + latest_sync_info = PromptService.is_sync_finished(sync_id) + if latest_sync_info.found == False or latest_sync_info.status == 'running': return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 20a1975..11c14db 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -320,50 +320,54 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): return {"detail": "The specific sync cannot be created."} -@router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 400: DetailSchema}) +@router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 500: DetailSchema}) def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): - data = payload.dict() - catalog = data["source"]["catalog"] - for stream in catalog["streams"]: - primary_key = [["id"]] - stream["primary_key"] = primary_key - stream["destination_sync_mode"] = "append_dedup" - # creating source credential - source_credential_payload = CredentialSchemaIn( - name="shopify", account=data["account"], connector_type=data["source"]["type"], - connector_config=data["source"]["config"]) - source_credential = create_credential(request, workspace_id, source_credential_payload) - # creating source - source_payload = SourceSchemaIn( - name="shopify", credential_id=source_credential.id, catalog=catalog) - source = create_source(request, workspace_id, source_payload) - workspace = Workspace.objects.get(id=workspace_id) - # creating default warehouse - storage_credentials = DefaultWarehouse.create(workspace) - source_access_info = {"source": source, "storage_credentials": storage_credentials} - SourceAccessInfo.objects.create(**source_access_info) - # creating destination credential - destination_credential_payload = CredentialSchemaIn( - name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) - destination_credential = create_credential(request, workspace_id, destination_credential_payload) - # creating destination - destination_payload = DestinationSchemaIn( - name="default warehouse", credential_id=destination_credential.id, catalog=catalog) - destination = create_destination(request, workspace_id, destination_payload) - data["source"] = Source.objects.get(id=source.id) - data["destination"] = Destination.objects.get(id=destination.id) - del data["account"] - if data["schedule"] is None: - schedule = {"run_interval": 3600000} - data["schedule"] = schedule - data["workspace"] = Workspace.objects.get(id=workspace_id) - if data["ui_state"] is None: - ui_state = {} - data["ui_state"] = ui_state - data["id"] = uuid.uuid4() - logger.debug(data) - sync = Sync.objects.create(**data) - return sync + try: + data = payload.dict() + catalog = data["source"]["catalog"] + for stream in catalog["streams"]: + primary_key = [["id"]] + stream["primary_key"] = primary_key + stream["destination_sync_mode"] = "append_dedup" + # creating source credential + source_credential_payload = CredentialSchemaIn( + name="shopify", account=data["account"], connector_type=data["source"]["type"], + connector_config=data["source"]["config"]) + source_credential = create_credential(request, workspace_id, source_credential_payload) + # creating source + source_payload = SourceSchemaIn( + name="shopify", credential_id=source_credential.id, catalog=catalog) + source = create_source(request, workspace_id, source_payload) + workspace = Workspace.objects.get(id=workspace_id) + # creating default warehouse + storage_credentials = DefaultWarehouse.create(workspace) + source_access_info = {"source": source, "storage_credentials": storage_credentials} + SourceAccessInfo.objects.create(**source_access_info) + # creating destination credential + destination_credential_payload = CredentialSchemaIn( + name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + destination_credential = create_credential(request, workspace_id, destination_credential_payload) + # creating destination + destination_payload = DestinationSchemaIn( + name="default warehouse", credential_id=destination_credential.id, catalog=catalog) + destination = create_destination(request, workspace_id, destination_payload) + data["source"] = Source.objects.get(id=source.id) + data["destination"] = Destination.objects.get(id=destination.id) + del data["account"] + if data["schedule"] is None: + schedule = {"run_interval": 3600000} + data["schedule"] = schedule + data["workspace"] = Workspace.objects.get(id=workspace_id) + if data["ui_state"] is None: + ui_state = {} + data["ui_state"] = ui_state + data["id"] = uuid.uuid4() + logger.debug(data) + sync = Sync.objects.create(**data) + return sync + except Exception: + logger.exception("Sync error") + return {"detail": "The specific sync cannot be created."} @router.post("/workspaces/{workspace_id}/syncs/update", response={200: SyncSchema, 400: DetailSchema}) diff --git a/core/schemas/explore.py b/core/schemas/explore.py index a594028..e4ce021 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -1,4 +1,5 @@ -from typing import Dict +from datetime import datetime +from typing import Dict, Optional from ninja import Field, ModelSchema, Schema from core.models import Explore from core.schemas.prompt import Filter, TimeWindow @@ -14,8 +15,8 @@ class ExploreSchemaOut(Schema): enabled: bool description: str workspace_id: str - last_sync_succeeded_at: str - last_sync_created_at: str + last_sync_succeeded_at: Optional[datetime] = None + last_sync_created_at: Optional[datetime] = None last_sync_result: str sync_state: str sync_id: str @@ -41,3 +42,9 @@ class ExploreSchemaIn(Schema): schema_id: str time_window: TimeWindow filters: list[Filter] + + +class LatestSyncInfo(Schema): + found: bool + status: Optional[str] = None + created_at: Optional[datetime] = None diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index 2503aa9..f32f5e8 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -1,3 +1,5 @@ +from datetime import datetime +from typing import Optional from ninja import Schema @@ -5,6 +7,7 @@ class TimeWindowRange(Schema): start: str end: str + class TimeWindow(Schema): label: str range: TimeWindowRange @@ -15,7 +18,7 @@ class Filter(Schema): operator: str name: str value: str - + class PromptPreviewSchemaIn(Schema): schema_id: str @@ -23,3 +26,6 @@ class PromptPreviewSchemaIn(Schema): filters: list[Filter] +class LastSuccessfulSyncInfo(Schema): + found: bool + run_end_at: Optional[datetime] = None diff --git a/core/services/explore.py b/core/services/explore.py index a43b4d5..dc7c0d6 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -13,6 +13,7 @@ import requests from core.routes.workspace_api import create_new_run from core.models import Credential, Destination, OAuthApiKeys, Prompt, Source, StorageCredentials, Sync, Workspace +from core.schemas.explore import LatestSyncInfo from core.schemas.prompt import Filter, TimeWindow from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -213,25 +214,13 @@ def create_run(request: object, workspace_id: str, sync_id: str, payload: object raise Exception("unable to create run") @staticmethod - def get_last_sync_successful_time(sync_id: str) -> str: - try: - response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") - json_string = response.content.decode('utf-8') - dict_data = json.loads(json_string) - return dict_data.get('run_end_at') if dict_data.get('found') == True else "" - except Exception as e: - logger.exception(f"Error : {e}") - raise Exception("unable to query activation") - - @staticmethod - def is_sync_created_or_running(sync_id: str) -> dict: + def get_latest_sync_info(sync_id: str) -> LatestSyncInfo: try: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/latest_sync_info") json_string = response.content.decode('utf-8') - dict_data = json.loads(json_string) - logger.debug(dict_data) - dict_data["is_running"] = dict_data.get('status') == 'running' - return dict_data + latest_sync_info_dict = json.loads(json_string) + latest_sync_info = LatestSyncInfo(**latest_sync_info_dict) + return latest_sync_info except Exception as e: logger.exception(f"Error : {e}") raise Exception("unable to query activation") diff --git a/core/services/prompts.py b/core/services/prompts.py index 2ad27cb..7526a98 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -5,10 +5,10 @@ import requests -from core.models import Credential, SourceAccessInfo, Sync +from core.models import Credential from liquid import Environment, FileSystemLoader, Mode, StrictUndefined -from core.schemas.prompt import TimeWindow, Filter +from core.schemas.prompt import LastSuccessfulSyncInfo, TimeWindow, Filter from decouple import config logger = logging.getLogger(__name__) ACTIVATION_URL = config("ACTIVATION_SERVER") @@ -55,15 +55,15 @@ def is_enabled(workspace_id: str, prompt: object) -> bool: return prompt.type in connector_types @staticmethod - def is_sync_finished(schema_id: str) -> bool: + def is_sync_finished(sync_id: str) -> LastSuccessfulSyncInfo: try: - source_access_info = SourceAccessInfo.objects.get(storage_credentials_id=schema_id) - sync = Sync.objects.get(source_id=source_access_info.source.id) - sync_id = sync.id response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") json_string = response.content.decode('utf-8') - dict_data = json.loads(json_string) - return dict_data["found"] == True + logger.debug(json_string) + last_success_sync_dict = json.loads(json_string) + last_success_sync = LastSuccessfulSyncInfo(**last_success_sync_dict) + logger.debug(last_success_sync) + return last_success_sync except Exception as e: logger.exception(e) raise e From f709dc66cf7dff30f836acbe132e35ac55b28e34 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 28 May 2024 13:46:50 +0530 Subject: [PATCH 107/159] feat: modified explore output schema --- core/routes/explore_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 4504863..be41ad9 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -51,8 +51,6 @@ def get_explores(request, workspace_id): last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) if last_successful_sync_info.found == True: explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at - else: - explore.last_sync_succeeded_at = "" return explores except Exception: logger.exception("explores listing error") From 3f232a110b351b5bfd815a26f96e0e842df3d4c5 Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Tue, 28 May 2024 14:24:58 +0530 Subject: [PATCH 108/159] Feat.minor fixes (#38) --- core/routes/prompt_api.py | 34 +++++++++++-------- core/schemas/prompt.py | 4 +++ core/services/explore.py | 21 +++++++----- core/services/prompt_templates/prompts.liquid | 2 +- core/services/prompts.py | 12 +++---- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index a21731b..cf025e4 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -1,18 +1,18 @@ import datetime import json import logging -import os from typing import List -from ninja import Router + import psycopg2 +from ninja import Router from pydantic import Json -from core.models import Credential, Prompt, Source, SourceAccessInfo, StorageCredentials, Sync -from core.schemas.prompt import PromptPreviewSchemaIn -from core.schemas.schemas import DetailSchema, PromptByIdSchema -from core.services.prompts import PromptService -from core.models import Prompt, StorageCredentials -from core.schemas.schemas import DetailSchema, PromptSchemaOut +from core.models import (Credential, Prompt, Source, SourceAccessInfo, + StorageCredentials, Sync) +from core.schemas.prompt import PromptPreviewSchemaIn, TableInfo +from core.schemas.schemas import (DetailSchema, PromptByIdSchema, + PromptSchemaOut) +from core.services.prompts import PromptService logger = logging.getLogger(__name__) router = Router() @@ -89,17 +89,23 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche sync_id = sync.id # checking wether sync has finished or not(from shopify to DB) latest_sync_info = PromptService.is_sync_finished(sync_id) - if latest_sync_info.found == False or latest_sync_info.status == 'running': - return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} + # if latest_sync_info.found == False or latest_sync_info.status == 'running': + # return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] - table_name = f'{schema_name}.{prompt.table}' - query = PromptService().build(table_name, prompt_req.time_window, prompt_req.filters) + table_info = TableInfo( + tableSchema= schema_name, + table= prompt.table + ) + + query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters) logger.debug(query) - host_url = os.environ["DATA_WAREHOUSE_URL"] + host = storage_credentials.connector_config.get('host') db_password = storage_credentials.connector_config.get('password') db_username = storage_credentials.connector_config.get('username') - conn = psycopg2.connect(host=host_url, port="5432", database="dvdrental", + database = storage_credentials.connector_config.get('database') + port = storage_credentials.connector_config.get('port') + conn = psycopg2.connect(host=host, port=port, database=database, user=db_username, password=db_password) cursor = conn.cursor() cursor.execute(query) diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index f32f5e8..cea88ce 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Optional + from ninja import Schema @@ -12,6 +13,9 @@ class TimeWindow(Schema): label: str range: TimeWindowRange +class TableInfo(Schema): + tableSchema: str + table: str class Filter(Schema): label: str diff --git a/core/services/explore.py b/core/services/explore.py index dc7c0d6..24ddf8f 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -6,16 +6,18 @@ from os.path import dirname, join from typing import List, Union +import requests +from decouple import config from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from os.path import dirname, join -from decouple import config -import requests + +from core.models import (Credential, Destination, OAuthApiKeys, Prompt, Source, + StorageCredentials, Sync, Workspace) from core.routes.workspace_api import create_new_run -from core.models import Credential, Destination, OAuthApiKeys, Prompt, Source, StorageCredentials, Sync, Workspace from core.schemas.explore import LatestSyncInfo -from core.schemas.prompt import Filter, TimeWindow +from core.schemas.prompt import Filter, TableInfo, TimeWindow from core.services.prompts import PromptService + logger = logging.getLogger(__name__) ACTIVATION_URL = config("ACTIVATION_SERVER") SPREADSHEET_SCOPES = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive'] @@ -96,8 +98,11 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte logger.debug(type(time_window)) prompt = Prompt.objects.get(id=prompt_id) namespace = storage_credential.connector_config["namespace"] - table = f'{namespace}.{prompt.table}' - query = PromptService().build(table, time_window, filters) + table_info = TableInfo( + tableSchema = namespace, + table = prompt.table + ) + query = PromptService().build(table_info, time_window, filters) # creating source cayalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" body = { @@ -124,7 +129,7 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte database = storage_credential.connector_config["database"] source_catalog["streams"][0]["stream"][ "name" - ] = f"{database}.{namespace}.{table}" + ] = f"{database}.{namespace}.{prompt.table}" source["catalog"] = source_catalog source["status"] = "active" logger.debug(source_catalog) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 1380c82..385a588 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -17,4 +17,4 @@ {% assign filterStr = filterStr | append: " and " %} {% endif %} -select * from {{ table }} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file +select * from "{{ schema }}"."{{ table }}" where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file diff --git a/core/services/prompts.py b/core/services/prompts.py index 7526a98..03ade44 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -3,13 +3,13 @@ from pathlib import Path import requests - +from decouple import config +from liquid import Environment, FileSystemLoader, Mode, StrictUndefined from core.models import Credential -from liquid import Environment, FileSystemLoader, Mode, StrictUndefined +from core.schemas.prompt import (Filter, LastSuccessfulSyncInfo, TableInfo, + TimeWindow) -from core.schemas.prompt import LastSuccessfulSyncInfo, TimeWindow, Filter -from decouple import config logger = logging.getLogger(__name__) ACTIVATION_URL = config("ACTIVATION_SERVER") @@ -20,7 +20,7 @@ def getTemplateFile(cls): return 'prompts.liquid' @staticmethod - def build(table, timeWindow: TimeWindow, filters: list[Filter]) -> str: + def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: try: if isinstance(timeWindow, TimeWindow): timeWindowDict = timeWindow.dict() @@ -41,7 +41,7 @@ def build(table, timeWindow: TimeWindow, filters: list[Filter]) -> str: ) template = env.get_template(file_name) filters = list(filters) - return template.render(table=table, timeWindow=timeWindowDict, filters=filterList) + return template.render(table=tableInfo.table, schema=tableInfo.tableSchema, timeWindow=timeWindowDict, filters=filterList) except Exception as e: logger.exception(e) raise e From a3d21a542fff228986f4e31a01408c5b20a689ff Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 29 May 2024 11:06:48 +0530 Subject: [PATCH 109/159] feat: Added ssl field while creating default destination --- core/services/warehouse_credentials.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py index c4ff484..2880b25 100644 --- a/core/services/warehouse_credentials.py +++ b/core/services/warehouse_credentials.py @@ -9,28 +9,32 @@ from core.models import StorageCredentials, Workspace logger = logging.getLogger(__name__) + class DefaultWarehouse(): @staticmethod - def create(workspace:object)->object: + def create(workspace: object) -> object: try: host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] db_username = os.environ["DATA_WAREHOUSE_USERNAME"] - conn = psycopg2.connect(host=host_url,port="5432",database="dvdrental",user=db_username,password=db_password) + conn = psycopg2.connect(host=host_url, port="5432", database="dvdrental", + user=db_username, password=db_password) cursor = conn.cursor() logger.debug("logger in creating new creds") user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) password = ''.join(random.choices(string.ascii_uppercase, k=17)) - creds = {'username': user_name, 'password': password,'namespace': user_name,'schema': user_name,'host':'classspace.in','database':'dvdrental','port':5432} + creds = {'username': user_name, 'password': password, 'namespace': user_name, + 'schema': user_name, 'host': 'classspace.in', 'database': 'dvdrental', 'port': 5432, 'ssl': False} credential_info = {"id": uuid.uuid4()} credential_info["workspace"] = Workspace.objects.get(id=workspace.id) credential_info["connector_config"] = creds result = StorageCredentials.objects.create(**credential_info) query = ("CREATE ROLE {username} LOGIN PASSWORD %s").format(username=user_name) cursor.execute(query, (password,)) - query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name = user_name) + query = ("CREATE SCHEMA AUTHORIZATION {name}").format(name=user_name) cursor.execute(query) - query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format(schema=user_name,username=user_name) + query = ("GRANT INSERT, UPDATE, SELECT ON ALL TABLES IN SCHEMA {schema} TO {username}").format( + schema=user_name, username=user_name) cursor.execute(query) query = ("ALTER USER {username} WITH SUPERUSER").format(username=user_name) cursor.execute(query) @@ -40,4 +44,3 @@ def create(workspace:object)->object: except Exception as e: logger.exception(e) raise Exception("Could not create warehouse credentials") - \ No newline at end of file From 33dee5110c1e340855d7266047454568ba36b89f Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 30 May 2024 11:17:18 +0530 Subject: [PATCH 110/159] feat: Added query in src postgres config for RETL flow --- core/services/explore.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index 24ddf8f..af5eef0 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -77,13 +77,14 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte credential["status"] = "active" storage_credential = StorageCredentials.objects.get(id=schema_id) connector_config = { - "ssl": False, + "ssl": storage_credential.connector_config["ssl"], "host": storage_credential.connector_config["host"], "port": storage_credential.connector_config["port"], "user": storage_credential.connector_config["username"], "database": storage_credential.connector_config["database"], "password": storage_credential.connector_config["password"], "namespace": storage_credential.connector_config["namespace"], + "query": query } credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) @@ -99,26 +100,16 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte prompt = Prompt.objects.get(id=prompt_id) namespace = storage_credential.connector_config["namespace"] table_info = TableInfo( - tableSchema = namespace, - table = prompt.table + tableSchema=namespace, + table=prompt.table ) query = PromptService().build(table_info, time_window, filters) # creating source cayalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" - body = { - "ssl": False, - "host": storage_credential.connector_config["host"], - "port": storage_credential.connector_config["port"], - "user": storage_credential.connector_config["username"], - "database": storage_credential.connector_config["database"], - "password": storage_credential.connector_config["password"], - "namespace": storage_credential.connector_config["namespace"], - "query": query - } config = { 'docker_image': 'valmiio/source-postgres', 'docker_tag': 'latest', - 'config': body + 'config': connector_config } response = requests.post(url, json=config) response_json = response.json() From 9c86e345a745913ed328c6ed3e329cdf7a533d49 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 30 May 2024 11:18:18 +0530 Subject: [PATCH 111/159] feat: Added query in src postgres config for RETL flow --- core/services/explore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/explore.py b/core/services/explore.py index af5eef0..236853b 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -76,6 +76,7 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte credential["account"] = account credential["status"] = "active" storage_credential = StorageCredentials.objects.get(id=schema_id) + query = PromptService().build(table_info, time_window, filters) connector_config = { "ssl": storage_credential.connector_config["ssl"], "host": storage_credential.connector_config["host"], @@ -103,7 +104,6 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte tableSchema=namespace, table=prompt.table ) - query = PromptService().build(table_info, time_window, filters) # creating source cayalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" config = { From 7013e3f697c2d37b90ee637941adeeccab3758d6 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 30 May 2024 11:44:26 +0530 Subject: [PATCH 112/159] feat: Added query in source postgres config for RETL flow --- core/services/explore.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index 236853b..cebdbed 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -75,7 +75,15 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte credential["name"] = "SRC_POSTGRES" credential["account"] = account credential["status"] = "active" + # building query + logger.debug(type(time_window)) + prompt = Prompt.objects.get(id=prompt_id) storage_credential = StorageCredentials.objects.get(id=schema_id) + namespace = storage_credential.connector_config["namespace"] + table_info = TableInfo( + tableSchema=namespace, + table=prompt.table + ) query = PromptService().build(table_info, time_window, filters) connector_config = { "ssl": storage_credential.connector_config["ssl"], @@ -96,14 +104,7 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte # creating source object source["workspace"] = Workspace.objects.get(id=workspace_id) source["credential"] = Credential.objects.get(id=cred.id) - # building query - logger.debug(type(time_window)) - prompt = Prompt.objects.get(id=prompt_id) namespace = storage_credential.connector_config["namespace"] - table_info = TableInfo( - tableSchema=namespace, - table=prompt.table - ) # creating source cayalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" config = { From 31158205ff685b0b142140870b857b39084f4ed6 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Thu, 30 May 2024 11:45:06 +0530 Subject: [PATCH 113/159] change prompt def file --- init_db/prompt_def.json | 200 ++++++++++++++++------------------- init_db/prompt_init.py | 10 +- init_db/test_prompt_def.json | 125 +++++++++++++++------- 3 files changed, 183 insertions(+), 152 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 7c1760c..4a4f4f0 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -1,110 +1,96 @@ { - "definitions":[ - { - - "name":"Inventory snapshot", - "description":"Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "query":"select * from dbt_transformer_p0.orders_with_product_data", - "table":"orders_with_product_data", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - - "name":"At Risk customers", - "description":"Top 100 customers who have not purchased from you in the last 30 days, ordered by their total purchase value.", - "query":"SELECT * FROM _airbyte_raw_customers", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - - "name":"Cart abandonment", - "description":"Get a snapshot of customers who have initiated the process of making a purchase on your platform but have left before completing the transaction.", - "query":"SELECT * FROM _airbyte_raw_checkouts", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - - "name":"Sales by Payment method", - "description":"Understand which payment methods your customers prefer. If a customer uses multiple payment methods for the same order, the order count and order value will be counted multiple times.", - "query":"SELECT * FROM _airbyte_raw_orders", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - - "name":"Order export with products", - "description":"Receive an export of your orders with line items. Use this report to share with your fulfillment teams and identify preorder volumes.", - "query":"SELECT * FROM _airbyte_raw_orders", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - - "name":"Average Order value", - "description":"Shows the average order value of all orders (excluding gift cards), divided by the total number of orders that contained at least one product other than a gift card. Order value includes taxes, shipping, and discounts before returns.", - "query":"SELECT * FROM _airbyte_raw_orders", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - "name":"Orders", - "description":"Orders, total sales, and products sold over the specified time period. Orders include all statuses. Refunded value is removed from Total Sales on the day of the order.", - "query":"SELECT * FROM _airbyte_raw_orders", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - "name":"Profit by product variant over time", - "description":"Review your profits by each variant over time. Use this report to identify high margin products and loss leaders. If collaborating on products on a profit share basis, use this to help calculate payouts.", - "query":"SELECT * FROM _airbyte_raw_products", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true - }, - { - "name":"New customer sales", - "description":"Sales metrics for new customers over the specified time-period.", - "query":"SELECT * FROM _airbyte_raw_customers", - "table":"orders_with_product_", - "parameters":{ - "test":"test" - }, - "package_id":"P0", - "gated":true + "definitions": [ + { + "name": "Inventory snapshot", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "table": "orders_with_product_data", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } } - - ] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Abandoned customers", + "description": "List of users who have abandoned their cart", + "table": "abandon_customers", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + } + ] } diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index 4d4276d..71597de 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -6,16 +6,18 @@ """ -from os.path import dirname, join import json +import logging +import os import uuid +from os.path import dirname, join + import requests -import os from requests.auth import HTTPBasicAuth -import logging + logger = logging.getLogger(__name__) -prompt_defs = json.loads(open(join(dirname(__file__), "test_prompt_def.json"), "r").read()) +prompt_defs = json.loads(open(join(dirname(__file__), "prompt_def.json"), "r").read()) for prompt_def in prompt_defs["definitions"]: logger.debug(prompt_def) diff --git a/init_db/test_prompt_def.json b/init_db/test_prompt_def.json index d2f1ccc..0f157e5 100644 --- a/init_db/test_prompt_def.json +++ b/init_db/test_prompt_def.json @@ -1,50 +1,93 @@ { "definitions": [ - { - "name": "Inventory snapshot", - "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "table": "orders_with_product_data", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", + { + "name": "Inventory snapshot", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "table": "orders_with_product_data", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { "type": "object", - "required": ["timeWindows", "sources"], "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] - }, - "timeWindow": { - "type": "object", - "properties": { - "label": {"type": "string"}, - "range": { - "type": "object", - "properties": { - "start": {"type": "string"}, - "end": {"type": "string"} - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": {"type": "string"}, - "name": {"type": "string"}, - "type": {"type": "string"}, - "value": {"type": "string"}, - "operator": {"type": "string", "enum": ["=", "!="]} - }, - "required": ["label", "name", "type", "value", "operator"] - } - } + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Abandoned customers", + "description": "List of users who have abandoned their cart", + "table": "abandon_customers", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows", "sources"], + "properties": { + "sourceId": { + "type": "string", + "enum": ["123", "321"] + }, + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } } + } }, - "type": "SRC_SHOPIFY", - "package_id": "P0", - "gated": true + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } } + } ] } From 20118662e0c7e8425945abd4a4ea8521d2eab725 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 30 May 2024 12:26:10 +0530 Subject: [PATCH 114/159] feat: replaced preview labels with now function in liquid template for preview --- core/services/prompt_templates/prompts.liquid | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 385a588..7f43aad 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -17,4 +17,8 @@ {% assign filterStr = filterStr | append: " and " %} {% endif %} -select * from "{{ schema }}"."{{ table }}" where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file +{% if timeWindow.range.start contains "now()" %} + select * from "{{ schema }}"."{{ table }}" where {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} +{% else %} + select * from "{{ schema }}"."{{ table }}" where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' +{% endif %} From 9932383e805c112b451527e135a28491b57c3ba2 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Thu, 30 May 2024 13:27:11 +0530 Subject: [PATCH 115/159] build prompt definitions using dbt models --- core/migrations/0030_auto_20240530_0701.py | 22 +++ core/models.py | 2 +- core/schemas/schemas.py | 10 +- core/services/prompt_templates/prompts.liquid | 2 +- init_db/prompt_def.json | 180 +++++++++++++++++- 5 files changed, 201 insertions(+), 15 deletions(-) create mode 100644 core/migrations/0030_auto_20240530_0701.py diff --git a/core/migrations/0030_auto_20240530_0701.py b/core/migrations/0030_auto_20240530_0701.py new file mode 100644 index 0000000..677a1f9 --- /dev/null +++ b/core/migrations/0030_auto_20240530_0701.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.5 on 2024-05-30 07:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0029_auto_20240508_0844'), + ] + + operations = [ + migrations.RemoveField( + model_name='prompt', + name='table', + ), + migrations.AddField( + model_name='prompt', + name='query', + field=models.CharField(default='query', max_length=1000), + ), + ] diff --git a/core/models.py b/core/models.py index 1ac0bec..d85233f 100644 --- a/core/models.py +++ b/core/models.py @@ -145,7 +145,7 @@ class Prompt(models.Model): description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") type = models.CharField(null=False, blank = False,max_length=256, default="SRC_SHOPIFY") spec = models.JSONField(blank=False, null=True) - table = models.CharField(max_length=256,null=False, blank=False,default="table_name") + query = models.CharField(max_length=1000,null=False, blank=False,default="query") package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 8420e2e..da96047 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -8,10 +8,14 @@ from datetime import datetime from typing import Dict, List, Optional + from django.contrib.auth.models import User from ninja import Field, ModelSchema, Schema from pydantic import UUID4 -from core.models import Account, Connector, Credential, Destination, Organization, Package, Prompt, Source, Sync, Workspace, OAuthApiKeys + +from core.models import (Account, Connector, Credential, Destination, + OAuthApiKeys, Organization, Package, Prompt, Source, + Sync, Workspace) def camel_to_snake(s): @@ -68,13 +72,13 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "table"] + model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "query"] class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "table"] + model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "query"] schemas: List[Dict] diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 385a588..1add67d 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -17,4 +17,4 @@ {% assign filterStr = filterStr | append: " and " %} {% endif %} -select * from "{{ schema }}"."{{ table }}" where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file +{{query}} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 4a4f4f0..49682b5 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -7,12 +7,8 @@ "spec": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "required": ["timeWindows", "sources"], + "required": ["timeWindows"], "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] - }, "timeWindow": { "type": "object", "properties": { @@ -49,16 +45,180 @@ { "name": "Abandoned customers", "description": "List of users who have abandoned their cart", - "table": "abandon_customers", + "query": "select * from {{schema}}.abandon_customers where {{filters}}", "spec": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "required": ["timeWindows", "sources"], + "required": ["timeWindows"], "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Average order value", + "description": "Average order value", + "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders where {{filters}}", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows"], + "properties": { + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Refunds by date", + "description": "List of users who have abandoned their cart", + "query": "select * from {{schema}}.abandon_customers where {{filters}}", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows"], + "properties": { + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Sales by category", + "description": "Sales by category", + "query": "select * from {{schema}}.abandon_customers where {{filters}}", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows"], + "properties": { + "timeWindow": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "range": { + "type": "object", + "properties": { + "start": { "type": "string" }, + "end": { "type": "string" } + } + } + } + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "name": { "type": "string" }, + "type": { "type": "string" }, + "value": { "type": "string" }, + "operator": { "type": "string", "enum": ["=", "!="] } + }, + "required": ["label", "name", "type", "value", "operator"] + } + } + } + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "Sales by payment method", + "description": "Sales by payment method", + "query": "SELECT transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) AS Total_Sales FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", + "spec": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["timeWindows"], + "properties": { "timeWindow": { "type": "object", "properties": { From 12bd8becdf1afea329e0027f1badc19e4b89417c Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Thu, 30 May 2024 16:54:14 +0530 Subject: [PATCH 116/159] Feat.prompt limit (#40) --- core/routes/prompt_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index cf025e4..e22369f 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -99,6 +99,7 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche ) query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters) + query = query + "limit 10" logger.debug(query) host = storage_credentials.connector_config.get('host') db_password = storage_credentials.connector_config.get('password') From 313c633850924ac27d883a4a918e48862d413158 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 30 May 2024 17:25:52 +0530 Subject: [PATCH 117/159] feat: filters can be applied on preview data --- core/routes/prompt_api.py | 6 +- core/schemas/prompt.py | 4 +- core/services/explore.py | 2 +- core/services/prompt_templates/prompts.liquid | 28 +++--- core/services/prompts.py | 11 ++- init_db/prompt_def.json | 2 +- init_db/prompt_init.py | 10 +- init_db/test_prompt_def.json | 93 ------------------- 8 files changed, 37 insertions(+), 119 deletions(-) delete mode 100644 init_db/test_prompt_def.json diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index cf025e4..5c707eb 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -94,8 +94,8 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] table_info = TableInfo( - tableSchema= schema_name, - table= prompt.table + tableSchema=schema_name, + query=prompt.query ) query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters) @@ -103,7 +103,7 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche host = storage_credentials.connector_config.get('host') db_password = storage_credentials.connector_config.get('password') db_username = storage_credentials.connector_config.get('username') - database = storage_credentials.connector_config.get('database') + database = storage_credentials.connector_config.get('database') port = storage_credentials.connector_config.get('port') conn = psycopg2.connect(host=host, port=port, database=database, user=db_username, password=db_password) diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index cea88ce..c72522b 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -13,9 +13,11 @@ class TimeWindow(Schema): label: str range: TimeWindowRange + class TableInfo(Schema): tableSchema: str - table: str + query: str + class Filter(Schema): label: str diff --git a/core/services/explore.py b/core/services/explore.py index cebdbed..3e30926 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -82,7 +82,7 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte namespace = storage_credential.connector_config["namespace"] table_info = TableInfo( tableSchema=namespace, - table=prompt.table + query=prompt.query ) query = PromptService().build(table_info, time_window, filters) connector_config = { diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 1add67d..11d0e8e 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -1,20 +1,22 @@ - {% assign filterStr = "" %} -{% for filter in filters %} - {% capture name -%} - "{{ filter.name }}" - {%- endcapture -%} - {% capture value -%} +{% for filter in filters %} + {% capture name %} + {{ filter.name }} + {% endcapture %} + {% capture value %} '{{ filter.value }}' - {%- endcapture -%} + {% endcapture %} {% assign operator = filter.operator %} - {% assign filterStr = filterStr| append: name | append: " " | append: operator | append: " "| append: value %} + {% assign filterStr = filterStr | append: name | append: " " | append: operator | append: " " | append: value %} {% if forloop.last == false %} - {% assign filterStr = filterStr| append: " and " %} + {% assign filterStr = filterStr | append: " and " %} {% endif %} {% endfor %} -{% if filterStr !="" %} +{% if filterStr != "" %} {% assign filterStr = filterStr | append: " and " %} -{% endif %} - -{{query}} where {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start}}' and "updated_at" <= '{{ timeWindow.range.end }}' \ No newline at end of file +{% endif %} +{% if timeWindow.range.start contains "now()" %} + {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} +{% else %} + {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' +{% endif %} diff --git a/core/services/prompts.py b/core/services/prompts.py index 03ade44..494975f 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,7 +1,7 @@ import json import logging from pathlib import Path - +from liquid import Template as LiquidTemplate import requests from decouple import config from liquid import Environment, FileSystemLoader, Mode, StrictUndefined @@ -41,7 +41,14 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) - ) template = env.get_template(file_name) filters = list(filters) - return template.render(table=tableInfo.table, schema=tableInfo.tableSchema, timeWindow=timeWindowDict, filters=filterList) + filterList = [filter.dict() for filter in filters] + rendered_query = template.render(filters=filterList, timeWindow=timeWindowDict) + liquid_template = LiquidTemplate(tableInfo.query) + context = { + "schema": tableInfo.tableSchema, + "filters": rendered_query + } + return liquid_template.render(context) except Exception as e: logger.exception(e) raise e diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 49682b5..af4a37b 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -3,7 +3,7 @@ { "name": "Inventory snapshot", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "table": "orders_with_product_data", + "query": "orders_with_product_data", "spec": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index 71597de..b546678 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -24,14 +24,14 @@ resp = requests.post( f"http://localhost:{os.environ['PORT']}/api/v1/superuser/prompts/create", json={ - "id":str(uuid.uuid4()), + "id": str(uuid.uuid4()), "name": prompt_def["name"], "description": prompt_def["description"], "type": prompt_def["type"], - "table":prompt_def["table"], - "spec":prompt_def["spec"], - "package_id":prompt_def["package_id"], - "gated":prompt_def["gated"], + "query": prompt_def["query"], + "spec": prompt_def["spec"], + "package_id": prompt_def["package_id"], + "gated": prompt_def["gated"], }, auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), ) diff --git a/init_db/test_prompt_def.json b/init_db/test_prompt_def.json deleted file mode 100644 index 0f157e5..0000000 --- a/init_db/test_prompt_def.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "definitions": [ - { - "name": "Inventory snapshot", - "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "table": "orders_with_product_data", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows", "sources"], - "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] - }, - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } - }, - "type": "SRC_SHOPIFY", - "package_id": "P0", - "gated": true - }, - { - "name": "Abandoned customers", - "description": "List of users who have abandoned their cart", - "table": "abandon_customers", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows", "sources"], - "properties": { - "sourceId": { - "type": "string", - "enum": ["123", "321"] - }, - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } - } - } - ] -} From 6f16d881cbd3cd2e3ebe332e891216863b36ea3a Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 31 May 2024 11:37:52 +0530 Subject: [PATCH 118/159] feat: Implement explore name uniqueness check and add table_name in PostgreSQL configuration --- core/routes/explore_api.py | 8 +++++++- core/routes/prompt_api.py | 2 +- core/services/explore.py | 19 ++++++++++++++++--- core/services/source_catalog.json | 2 +- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index be41ad9..e113565 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -61,6 +61,12 @@ def get_explores(request, workspace_id): def create_explore(request, workspace_id, payload: ExploreSchemaIn): data = payload.dict() try: + try: + ExploreService.check_name_uniquesness(data["name"], workspace_id) + except Exception as err: + logger.exception(err) + message = str(err) + return (400, {"detail": message}) data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) prompt = Prompt.objects.get(id=data["prompt_id"]) @@ -74,7 +80,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["account"] = account # create source source = ExploreService.create_source( - data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) + data["name"], data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) # create destination spreadsheet_name = f"valmi.io {prompt.name} sheet" destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 78c23b7..78bfa46 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -99,7 +99,7 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche ) query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters) - query = query + "limit 10" + query = query + " limit 10" logger.debug(query) host = storage_credentials.connector_config.get('host') db_password = storage_credentials.connector_config.get('password') diff --git a/core/services/explore.py b/core/services/explore.py index 3e30926..345a943 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -3,6 +3,7 @@ import logging import os import uuid +import re from os.path import dirname, join from typing import List, Union @@ -11,7 +12,7 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from core.models import (Credential, Destination, OAuthApiKeys, Prompt, Source, +from core.models import (Credential, Destination, Explore, OAuthApiKeys, Prompt, Source, StorageCredentials, Sync, Workspace) from core.routes.workspace_api import create_new_run from core.schemas.explore import LatestSyncInfo @@ -66,7 +67,7 @@ def create_spreadsheet(name: str, refresh_token: str) -> str: raise Exception("spreadhseet creation failed") @staticmethod - def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filters: list[Filter], workspace_id: str, account: object) -> object: + def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_window: TimeWindow, filters: list[Filter], workspace_id: str, account: object) -> object: try: # creating source credentail credential = {"id": uuid.uuid4()} @@ -85,6 +86,7 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte query=prompt.query ) query = PromptService().build(table_info, time_window, filters) + # creating source credentials connector_config = { "ssl": storage_credential.connector_config["ssl"], "host": storage_credential.connector_config["host"], @@ -93,7 +95,8 @@ def create_source(prompt_id: str, schema_id: str, time_window: TimeWindow, filte "database": storage_credential.connector_config["database"], "password": storage_credential.connector_config["password"], "namespace": storage_credential.connector_config["namespace"], - "query": query + "query": query, + "table_name": explore_table_name } credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) @@ -221,3 +224,13 @@ def get_latest_sync_info(sync_id: str) -> LatestSyncInfo: except Exception as e: logger.exception(f"Error : {e}") raise Exception("unable to query activation") + + @staticmethod + def check_name_uniquesness(name: str, workspace_id: str) -> bool: + if Explore.objects.filter(workspace_id=workspace_id, name=name).exists(): + raise Exception(f"The name '{name}' already exists. Please provide a different name.") + if re.match(r'^\d', name): + raise Exception(f"Explore name cannot start with a number.") + if re.search(r'[^a-zA-Z0-9]', name): + raise Exception(f"Explore name cannot contain special characters.") + return True diff --git a/core/services/source_catalog.json b/core/services/source_catalog.json index 3df6484..722d342 100644 --- a/core/services/source_catalog.json +++ b/core/services/source_catalog.json @@ -4,7 +4,7 @@ "id_key": "id", "stream": { }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "append" } ] From f5b5b8406ece809def6788f4572100bfa0c54c92 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 31 May 2024 13:35:37 +0530 Subject: [PATCH 119/159] feat: Added Decimal type in josn rendering for preview data --- core/routes/prompt_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 78bfa46..d08731b 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -1,4 +1,5 @@ import datetime +from decimal import Decimal import json import logging from typing import List @@ -74,6 +75,8 @@ def get_prompt_by_id(request, workspace_id, prompt_id): def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() + if isinstance(obj, Decimal): + return str(obj) @router.post("/workspaces/{workspace_id}/prompts/{prompt_id}/preview", response={200: Json, 400: DetailSchema}) @@ -113,6 +116,7 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche items = [dict(zip([key[0] for key in cursor.description], row)) for row in cursor.fetchall()] conn.commit() conn.close() + logger.debug(items) return json.dumps(items, indent=4, default=custom_serializer) except Exception as err: logger.exception(f"preview fetching error:{err}") From 30fcf9fc6f7c9cb662200e535e3a808e1a81dba2 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 4 Jun 2024 15:47:27 +0530 Subject: [PATCH 120/159] feat: replaced table name with explore name in explore creation --- core/services/explore.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index 345a943..418a653 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -95,8 +95,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ "database": storage_credential.connector_config["database"], "password": storage_credential.connector_config["password"], "namespace": storage_credential.connector_config["namespace"], - "query": query, - "table_name": explore_table_name + "query": query } credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) @@ -124,7 +123,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ database = storage_credential.connector_config["database"] source_catalog["streams"][0]["stream"][ "name" - ] = f"{database}.{namespace}.{prompt.table}" + ] = f"{database}.{namespace}.{explore_table_name}" source["catalog"] = source_catalog source["status"] = "active" logger.debug(source_catalog) From a2092b43df1677142911ad6444eb8800092d9df2 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Thu, 6 Jun 2024 12:16:27 +0530 Subject: [PATCH 121/159] remove break lines from liquid template --- core/services/prompt_templates/prompts.liquid | 12 ++-- init_db/prompt_def.json | 2 +- init_db/test.json | 65 +++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 init_db/test.json diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 11d0e8e..22a5a13 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -1,4 +1,4 @@ -{% assign filterStr = "" %} +{%- assign filterStr = "" -%} {% for filter in filters %} {% capture name %} {{ filter.name }} @@ -12,11 +12,11 @@ {% assign filterStr = filterStr | append: " and " %} {% endif %} {% endfor %} -{% if filterStr != "" %} +{%- if filterStr != "" -%} {% assign filterStr = filterStr | append: " and " %} -{% endif %} -{% if timeWindow.range.start contains "now()" %} +{%- endif -%} +{%- if timeWindow.range.start contains "now()" -%} {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} -{% else %} +{%- else -%} {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' -{% endif %} +{%- endif -%} diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index af4a37b..ebeb0bf 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -213,7 +213,7 @@ { "name": "Sales by payment method", "description": "Sales by payment method", - "query": "SELECT transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) AS Total_Sales FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", + "query": "SELECT transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales, md5(row(payment_method, total_orders, total_sales)) as id FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", "spec": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/init_db/test.json b/init_db/test.json new file mode 100644 index 0000000..c4c0b0a --- /dev/null +++ b/init_db/test.json @@ -0,0 +1,65 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["time_window"], + "properties": { + "time_window": { + "type": "object", + "properties": { + "label": { + "type": "string", + "enum": [ + "custom", + "Last 7 days", + "Last 15 days", + "Last 30 days", + "Last 60 days", + "Last 90 days" + ] + }, + "range": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time" + }, + "end": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "if": { + "properties": { + "time_window": { + "properties": { + "label": { + "const": "Last 7 days" + } + } + } + } + }, + "then": { + "properties": { + "time_window": { + "properties": { + "range": { + "properties": { + "start": { + "const": "now() - 7 d" + }, + "end": { + "const": "now()" + } + } + } + } + } + } + } +} From 889c2bae4f379c0e24c7d3ee2093e7d79273a343 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Thu, 6 Jun 2024 12:24:55 +0530 Subject: [PATCH 122/159] add md5 to hash column in sales by payment method --- init_db/prompt_def.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index ebeb0bf..a4de0d6 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -213,7 +213,7 @@ { "name": "Sales by payment method", "description": "Sales by payment method", - "query": "SELECT transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales, md5(row(payment_method, total_orders, total_sales)) as id FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", + "query": "SELECT md5(row(transactions.gateway, COUNT(*)::text, ROUND(SUM(total_price))::text)::text) as id, transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", "spec": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", From 07209ec6a6cc435291b550dc4086e912ffc20f56 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Thu, 6 Jun 2024 12:52:37 +0530 Subject: [PATCH 123/159] feat: changed the spreadsheet creation to private --- core/services/explore.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index 418a653..3765384 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -49,16 +49,16 @@ def create_spreadsheet(name: str, refresh_token: str) -> str: ) spreadsheet_id = spreadsheet.get("spreadsheetId") # Update the sharing settings to make the spreadsheet publicly accessible - drive_service = build('drive', 'v3', credentials=credentials) - drive_service.permissions().create( - fileId=spreadsheet_id, - body={ - "role": "writer", - "type": "anyone", - "withLink": True - }, - fields="id" - ).execute() + # drive_service = build('drive', 'v3', credentials=credentials) + # drive_service.permissions().create( + # fileId=spreadsheet_id, + # body={ + # "role": "writer", + # "type": "anyone", + # "withLink": True + # }, + # fields="id" + # ).execute() spreadsheet_url = f"{base_spreadsheet_url}{spreadsheet_id}" return spreadsheet_url From de0463e3f1363495029869d47713fa34850af7c9 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 7 Jun 2024 15:46:31 +0530 Subject: [PATCH 124/159] feat: Added field connected to get no of connected connectors for workspace id --- core/routes/api.py | 3 ++- core/routes/connector_api.py | 20 +++++++++++--------- core/schemas/schemas.py | 7 +++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/routes/api.py b/core/routes/api.py index 366f7cf..9dfa94a 100644 --- a/core/routes/api.py +++ b/core/routes/api.py @@ -148,8 +148,9 @@ def list_spaces(request): router.add_router("superuser/", superuser_api_router, auth=[BasicAuth()]) router.add_router("streams/", stream_api_router, tags=["streams"]) router.add_router("oauth/", oauth_api_router, tags=["oauth"]) -router.add_router("connectors", connector_api_router, tags=["connectors"]) + router.add_router("packages", package_api_router, tags=["packages"]) router.add_router("", workspace_api_router, tags=["workspaces"]) router.add_router("", prompt_api_router, tags=["prompts"]) router.add_router("", explore_api_router, tags=["explores"]) +router.add_router("", connector_api_router, tags=["connectors"]) diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py index caf67c8..c61533d 100644 --- a/core/routes/connector_api.py +++ b/core/routes/connector_api.py @@ -3,9 +3,9 @@ from ninja import Router -from core.schemas.schemas import ConnectorSchema, DetailSchema +from core.schemas.schemas import ConnectorOutputSchema, ConnectorSchema, DetailSchema -from core.models import Connector, OAuthApiKeys, Workspace +from core.models import Connector, Credential, OAuthApiKeys, Workspace router = Router() @@ -13,32 +13,34 @@ logger = logging.getLogger(__name__) -@router.get("/", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) -def get_connectors(request): +@router.get("{workspace_id}/connectors", response={200: Dict[str, List[ConnectorOutputSchema]], 400: DetailSchema}) +def get_connectors(request, workspace_id): # check for admin permissions try: logger.debug("listing connectors") - connectors = Connector.objects.all() - + connectors = Connector.objects.filter(mode=['etl'], type__startswith='S') logger.info(f"connectors - {connectors}") src_dst_dict: Dict[str, List[ConnectorSchema]] = {} src_dst_dict["SRC"] = [] src_dst_dict["DEST"] = [] for conn in connectors: logger.info(f"conn{conn}") + conn.connected = Credential.objects.filter(workspace_id=workspace_id, connector_id=conn.type).count() arr = conn.type.split("_") if arr[0] == "SRC": + logger.debug("inside src") src_dst_dict["SRC"].append(conn) elif arr[0] == "DEST": src_dst_dict["DEST"].append(conn) + logger.debug(src_dst_dict) return src_dst_dict except Exception: logger.exception("connector listing error") return (400, {"detail": "The list of connectors cannot be fetched."}) -@router.get("{workspace_id}/configured", response={200: Dict[str, List[ConnectorSchema]], - 400: DetailSchema}) +@router.get("{workspace_id}/connectors/configured", response={200: Dict[str, List[ConnectorSchema]], + 400: DetailSchema}) def get_connectors_configured(request, workspace_id): try: @@ -75,7 +77,7 @@ def get_connectors_configured(request, workspace_id): return (400, {"detail": "The list of connectors cannot be fetched."}) -@router.get("{workspace_id}/not-configured", +@router.get("{workspace_id}/connectors/not-configured", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) def get_connectors_not_configured(request, workspace_id): diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index da96047..dac7d4e 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -63,6 +63,13 @@ class Config(CamelSchemaConfig): model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] +class ConnectorOutputSchema(ModelSchema): + class Config(CamelSchemaConfig): + model = Connector + model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] + connected: str + + class PackageSchema(ModelSchema): class Config(CamelSchemaConfig): model = Package From 95171101f67444ef1b34181ea1946abaa56cef54 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 7 Jun 2024 15:50:07 +0530 Subject: [PATCH 125/159] feat: Modified connected field in connector output schema to int --- core/schemas/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index dac7d4e..4dbb9ba 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -67,7 +67,7 @@ class ConnectorOutputSchema(ModelSchema): class Config(CamelSchemaConfig): model = Connector model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] - connected: str + connected: int class PackageSchema(ModelSchema): From c702636223387e3d421a311260396d4109cab443 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 7 Jun 2024 15:59:29 +0530 Subject: [PATCH 126/159] feat: Added default database in .env --- .env-example | 3 ++- core/routes/connector_api.py | 2 +- core/schemas/schemas.py | 2 +- core/services/warehouse_credentials.py | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.env-example b/.env-example index 2bd8d9a..551713a 100644 --- a/.env-example +++ b/.env-example @@ -45,4 +45,5 @@ OTEL_EXPORTER_OTLP_INSECURE=True DATA_WAREHOUSE_URL="********************" DATA_WAREHOUSE_USERNAME="****************" -DATA_WAREHOUSE_PASSWORD="***************8" \ No newline at end of file +DATA_WAREHOUSE_PASSWORD="***************8" +DATA_WAREHOUSE_DB_NAME="******************" \ No newline at end of file diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py index c61533d..41d73f7 100644 --- a/core/routes/connector_api.py +++ b/core/routes/connector_api.py @@ -25,7 +25,7 @@ def get_connectors(request, workspace_id): src_dst_dict["DEST"] = [] for conn in connectors: logger.info(f"conn{conn}") - conn.connected = Credential.objects.filter(workspace_id=workspace_id, connector_id=conn.type).count() + conn.connections = Credential.objects.filter(workspace_id=workspace_id, connector_id=conn.type).count() arr = conn.type.split("_") if arr[0] == "SRC": logger.debug("inside src") diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 4dbb9ba..b975ac8 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -67,7 +67,7 @@ class ConnectorOutputSchema(ModelSchema): class Config(CamelSchemaConfig): model = Connector model_fields = ["type", "docker_image", "docker_tag", "display_name", "oauth", "oauth_keys", "mode"] - connected: int + connections: int class PackageSchema(ModelSchema): diff --git a/core/services/warehouse_credentials.py b/core/services/warehouse_credentials.py index 2880b25..028b7bf 100644 --- a/core/services/warehouse_credentials.py +++ b/core/services/warehouse_credentials.py @@ -17,14 +17,15 @@ def create(workspace: object) -> object: host_url = os.environ["DATA_WAREHOUSE_URL"] db_password = os.environ["DATA_WAREHOUSE_PASSWORD"] db_username = os.environ["DATA_WAREHOUSE_USERNAME"] - conn = psycopg2.connect(host=host_url, port="5432", database="dvdrental", + database = os.environ["DATA_WAREHOUSE_DB_NAME"] + conn = psycopg2.connect(host=host_url, port="5432", database=database, user=db_username, password=db_password) cursor = conn.cursor() logger.debug("logger in creating new creds") user_name = ''.join(random.choices(string.ascii_lowercase, k=17)) password = ''.join(random.choices(string.ascii_uppercase, k=17)) creds = {'username': user_name, 'password': password, 'namespace': user_name, - 'schema': user_name, 'host': 'classspace.in', 'database': 'dvdrental', 'port': 5432, 'ssl': False} + 'schema': user_name, 'host': host_url, 'database': database, 'port': 5432, 'ssl': False} credential_info = {"id": uuid.uuid4()} credential_info["workspace"] = Workspace.objects.get(id=workspace.id) credential_info["connector_config"] = creds From 808f82ac94973f143f688a7c8ddafac3e7f0fa63 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 7 Jun 2024 16:23:47 +0530 Subject: [PATCH 127/159] feat: Fetch only ETL sources for configured endpoint --- core/routes/connector_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py index 41d73f7..02624b0 100644 --- a/core/routes/connector_api.py +++ b/core/routes/connector_api.py @@ -53,6 +53,8 @@ def get_connectors_configured(request, workspace_id): connectors = Connector.objects.filter( oauth=True, oauth_keys="private", + mode=['etl'], + type__startswith='S', type__in=configured_connectors) src_dst_dict: Dict[str, List[ConnectorSchema]] = {} @@ -90,7 +92,9 @@ def get_connectors_not_configured(request, workspace_id): connectors = Connector.objects.filter( oauth=True, - oauth_keys="private" + oauth_keys="private", + mode=['etl'], + type__startswith='S' ).exclude(type__in=configured_connectors) src_dst_dict: Dict[str, List[ConnectorSchema]] = {} From cbf2b13a3c746f6026b1d390ad1e5054df35df65 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 7 Jun 2024 16:23:58 +0530 Subject: [PATCH 128/159] feat: Fetch only sources for configured endpoint --- core/routes/connector_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/routes/connector_api.py b/core/routes/connector_api.py index 02624b0..6e0ce97 100644 --- a/core/routes/connector_api.py +++ b/core/routes/connector_api.py @@ -79,7 +79,7 @@ def get_connectors_configured(request, workspace_id): return (400, {"detail": "The list of connectors cannot be fetched."}) -@router.get("{workspace_id}/connectors/not-configured", +@router.get("{workspace_id}/connectors/not_configured", response={200: Dict[str, List[ConnectorSchema]], 400: DetailSchema}) def get_connectors_not_configured(request, workspace_id): From 360510ee67cf3593180a851d622132273915f040 Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Fri, 7 Jun 2024 17:20:45 +0530 Subject: [PATCH 129/159] Feat.add default spreadsheetname (#43) --- core/services/explore.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/services/explore.py b/core/services/explore.py index 3765384..7397b54 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -2,8 +2,8 @@ import json import logging import os -import uuid import re +import uuid from os.path import dirname, join from typing import List, Union @@ -12,8 +12,8 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from core.models import (Credential, Destination, Explore, OAuthApiKeys, Prompt, Source, - StorageCredentials, Sync, Workspace) +from core.models import (Credential, Destination, Explore, OAuthApiKeys, + Prompt, Source, StorageCredentials, Sync, Workspace) from core.routes.workspace_api import create_new_run from core.schemas.explore import LatestSyncInfo from core.schemas.prompt import Filter, TableInfo, TimeWindow @@ -41,7 +41,17 @@ def create_spreadsheet(name: str, refresh_token: str) -> str: ) service = build("sheets", "v4", credentials=credentials) # Create the spreadsheet - spreadsheet = {"properties": {"title": name}} + spreadsheet = { + "properties": { + "title": name, + }, + "sheets": [ + { + "properties": { + "title": name + } + } + ]} spreadsheet = ( service.spreadsheets() .create(body=spreadsheet, fields="spreadsheetId") From fffeef30d6e817a0f5c02b372dfea878d08a691c Mon Sep 17 00:00:00 2001 From: Ganesh varma Date: Fri, 7 Jun 2024 17:47:15 +0530 Subject: [PATCH 130/159] add default spreadsheet name1 (#44) --- core/routes/explore_api.py | 4 ++-- core/services/explore.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index e113565..06b0b62 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -82,8 +82,8 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): source = ExploreService.create_source( data["name"], data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) # create destination - spreadsheet_name = f"valmi.io {prompt.name} sheet" - destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) + spreadsheet_title = f"valmi.io {prompt.name} sheet" + destination_data = ExploreService.create_destination(spreadsheet_title, data["name"], workspace_id, account) spreadsheet_url = destination_data[0] destination = destination_data[1] # create sync diff --git a/core/services/explore.py b/core/services/explore.py index 7397b54..93534a9 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -26,7 +26,7 @@ class ExploreService: @staticmethod - def create_spreadsheet(name: str, refresh_token: str) -> str: + def create_spreadsheet(title: str, name: str, refresh_token: str) -> str: logger.debug("create_spreadsheet") credentials_dict = { "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], @@ -43,7 +43,7 @@ def create_spreadsheet(name: str, refresh_token: str) -> str: # Create the spreadsheet spreadsheet = { "properties": { - "title": name, + "title": title, }, "sheets": [ { @@ -133,7 +133,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ database = storage_credential.connector_config["database"] source_catalog["streams"][0]["stream"][ "name" - ] = f"{database}.{namespace}.{explore_table_name}" + ] = explore_table_name source["catalog"] = source_catalog source["status"] = "active" logger.debug(source_catalog) @@ -144,7 +144,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ raise Exception("unable to create source") @staticmethod - def create_destination(spreadsheet_name: str, workspace_id: str, account: object) -> List[Union[str, object]]: + def create_destination(spreadsheet_title: str, spreadsheet_name: str, workspace_id: str, account: object) -> List[Union[str, object]]: try: # creating destination credential oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id, type="GOOGLE_LOGIN") @@ -155,7 +155,7 @@ def create_destination(spreadsheet_name: str, workspace_id: str, account: object credential["account"] = account credential["status"] = "active" spreadsheet_url = ExploreService.create_spreadsheet( - spreadsheet_name, refresh_token=oauthkeys.oauth_config["refresh_token"]) + spreadsheet_title,spreadsheet_name, refresh_token=oauthkeys.oauth_config["refresh_token"]) connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { From 3279564c87d6d5923c7c2b6e4267e47183644453 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 10 Jun 2024 16:01:36 +0530 Subject: [PATCH 131/159] feat: Allow users to specify custom spreadsheet URLs for explore creation --- core/routes/explore_api.py | 18 +++++++++-- core/schemas/explore.py | 1 + core/services/explore.py | 61 +++++++++++++++++++++++++++++++++----- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index e113565..5c75c36 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -60,13 +60,23 @@ def get_explores(request, workspace_id): @router.post("/workspaces/{workspace_id}/explores/create", response={200: ExploreSchema, 400: DetailSchema}) def create_explore(request, workspace_id, payload: ExploreSchemaIn): data = payload.dict() + logger.debug(data) try: try: - ExploreService.check_name_uniquesness(data["name"], workspace_id) + table_name = ExploreService.validate_explore_name(data["name"], workspace_id) except Exception as err: logger.exception(err) message = str(err) return (400, {"detail": message}) + # chceck if sheet_url is not emoty check for necessary permissions to write into file + if data["sheet_url"] is not None: + try: + if not ExploreService.is_sheet_accessible(data["sheet_url"], workspace_id): + return (400, {"detail": "You dont have access or missing write access to the spreadsheet"}) + except Exception as err: + logger.exception(err) + message = str(err) + return (400, {"detail": message}) data["id"] = uuid.uuid4() data["workspace"] = Workspace.objects.get(id=workspace_id) prompt = Prompt.objects.get(id=data["prompt_id"]) @@ -80,10 +90,11 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["account"] = account # create source source = ExploreService.create_source( - data["name"], data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) + table_name, data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) # create destination spreadsheet_name = f"valmi.io {prompt.name} sheet" - destination_data = ExploreService.create_destination(spreadsheet_name, workspace_id, account) + destination_data = ExploreService.create_destination( + spreadsheet_name, data.get("sheet_url"), workspace_id, account) spreadsheet_url = destination_data[0] destination = destination_data[1] # create sync @@ -92,6 +103,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): del data["schema_id"] del data["filters"] del data["time_window"] + del data["sheet_url"] # data["name"] = f"valmiio {prompt.name}" data["sync"] = sync data["ready"] = False diff --git a/core/schemas/explore.py b/core/schemas/explore.py index e4ce021..ca30363 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -37,6 +37,7 @@ class ExploreStatusIn(Schema): class ExploreSchemaIn(Schema): name: str + sheet_url: Optional[str] account: Dict = None prompt_id: str schema_id: str diff --git a/core/services/explore.py b/core/services/explore.py index 3765384..3f70513 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -107,7 +107,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ source["workspace"] = Workspace.objects.get(id=workspace_id) source["credential"] = Credential.objects.get(id=cred.id) namespace = storage_credential.connector_config["namespace"] - # creating source cayalog + # creating source catalog url = f"{ACTIVATION_URL}/connectors/SRC_POSTGRES/discover" config = { 'docker_image': 'valmiio/source-postgres', @@ -134,7 +134,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ raise Exception("unable to create source") @staticmethod - def create_destination(spreadsheet_name: str, workspace_id: str, account: object) -> List[Union[str, object]]: + def create_destination(spreadsheet_name: str, sheet_url: str, workspace_id: str, account: object) -> List[Union[str, object]]: try: # creating destination credential oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id, type="GOOGLE_LOGIN") @@ -144,8 +144,11 @@ def create_destination(spreadsheet_name: str, workspace_id: str, account: object credential["name"] = "DEST_GOOGLE-SHEETS" credential["account"] = account credential["status"] = "active" - spreadsheet_url = ExploreService.create_spreadsheet( - spreadsheet_name, refresh_token=oauthkeys.oauth_config["refresh_token"]) + if sheet_url is None: + spreadsheet_url = ExploreService.create_spreadsheet( + spreadsheet_name, refresh_token=oauthkeys.oauth_config["refresh_token"]) + else: + spreadsheet_url = sheet_url connector_config = { "spreadsheet_id": spreadsheet_url, "credentials": { @@ -225,11 +228,53 @@ def get_latest_sync_info(sync_id: str) -> LatestSyncInfo: raise Exception("unable to query activation") @staticmethod - def check_name_uniquesness(name: str, workspace_id: str) -> bool: + def validate_explore_name(name: str, workspace_id: str) -> str: if Explore.objects.filter(workspace_id=workspace_id, name=name).exists(): raise Exception(f"The name '{name}' already exists. Please provide a different name.") if re.match(r'^\d', name): raise Exception(f"Explore name cannot start with a number.") - if re.search(r'[^a-zA-Z0-9]', name): - raise Exception(f"Explore name cannot contain special characters.") - return True + name = re.sub(r'[^a-zA-Z0-9]', '_', name) + return name + + @staticmethod + def extract_spreadsheet_id(sheet_url: str) -> str: + pattern = r"/spreadsheets/d/([a-zA-Z0-9-_]+)" + match = re.search(pattern, sheet_url) + if match: + return match.group(1) + else: + return None + + @ staticmethod + # check wether the given sheet is accessible using the stored refresh token + def is_sheet_accessible(sheet_url: str, workspace_id: str) -> bool: + try: + oauthkeys = OAuthApiKeys.objects.get(workspace_id=workspace_id, type="GOOGLE_LOGIN") + credentials_dict = { + "client_id": os.environ["NEXTAUTH_GOOGLE_CLIENT_ID"], + "client_secret": os.environ["NEXTAUTH_GOOGLE_CLIENT_SECRET"], + "refresh_token": oauthkeys.oauth_config["refresh_token"] + } + credentials = Credentials.from_authorized_user_info( + credentials_dict, scopes=SPREADSHEET_SCOPES + ) + drive_service = build('drive', 'v3', credentials=credentials) + + spreadsheet_id = ExploreService.extract_spreadsheet_id(sheet_url) + + # Check if the file exists if file does not exist it will throw an exception + try: + response = drive_service.files().get(fileId=spreadsheet_id).execute() + except Exception as e: + return False + spreadsheet_metadata = drive_service.files().get(fileId=spreadsheet_id, fields='id, permissions').execute() + + # Check permissions to see if the file is accessible + permissions = spreadsheet_metadata.get('permissions', []) + for permission in permissions: + if (permission.get('type') == 'user' and permission.get('role') == 'writer') or (permission.get('type') == 'domain' and permission.get('role') == 'writer') or (permission.get('type') == 'anyone' and permission.get('role') == 'writer'): + return True + return False + except Exception as e: + logger.exception(f"Error : {e}") + raise Exception(e) From 37b3cc7091360c811016e5b3afb8c8738d649e12 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Mon, 10 Jun 2024 16:55:23 +0530 Subject: [PATCH 132/159] feat: changes for prompt def --- core/migrations/0031_remove_prompt_spec.py | 17 ++ core/migrations/0032_auto_20240607_1146.py | 23 ++ core/models.py | 7 +- core/routes/engine_api.py | 22 +- core/schemas/schemas.py | 4 +- init_db/prompt_def.json | 234 ++++----------------- init_db/prompt_init.py | 40 ++-- 7 files changed, 117 insertions(+), 230 deletions(-) create mode 100644 core/migrations/0031_remove_prompt_spec.py create mode 100644 core/migrations/0032_auto_20240607_1146.py diff --git a/core/migrations/0031_remove_prompt_spec.py b/core/migrations/0031_remove_prompt_spec.py new file mode 100644 index 0000000..596d008 --- /dev/null +++ b/core/migrations/0031_remove_prompt_spec.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1.5 on 2024-06-07 11:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0030_auto_20240530_0701'), + ] + + operations = [ + migrations.RemoveField( + model_name='prompt', + name='spec', + ), + ] diff --git a/core/migrations/0032_auto_20240607_1146.py b/core/migrations/0032_auto_20240607_1146.py new file mode 100644 index 0000000..c80ea63 --- /dev/null +++ b/core/migrations/0032_auto_20240607_1146.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.5 on 2024-06-07 11:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_remove_prompt_spec'), + ] + + operations = [ + migrations.AddField( + model_name='prompt', + name='filters', + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name='prompt', + name='operators', + field=models.JSONField(default={'integer': ['=', '>', '<', '>=', '<=', '!='], 'string': ['=', '!=', 'IN', 'NOT IN']}), + ), + ] diff --git a/core/models.py b/core/models.py index d85233f..0499eaf 100644 --- a/core/models.py +++ b/core/models.py @@ -144,7 +144,12 @@ class Prompt(models.Model): name = models.CharField(max_length=256, null=False, blank=False,unique=True) description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") type = models.CharField(null=False, blank = False,max_length=256, default="SRC_SHOPIFY") - spec = models.JSONField(blank=False, null=True) + # spec = models.JSONField(blank=False, null=True) + filters = models.JSONField(default=dict) + operators = models.JSONField(default={ + 'string': ["=", "!=", "IN", "NOT IN"], + 'integer': ["=", ">", "<", ">=", "<=", "!="] + }) query = models.CharField(max_length=1000,null=False, blank=False,default="query") package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) diff --git a/core/routes/engine_api.py b/core/routes/engine_api.py index 91994a5..ef0262a 100644 --- a/core/routes/engine_api.py +++ b/core/routes/engine_api.py @@ -25,7 +25,7 @@ from opentelemetry.metrics import get_meter_provider # from opentelemetry import trace - +from django.db import connection from valmi_app_backend.utils import replace_values_in_json router = Router() @@ -99,17 +99,27 @@ def create_connector(request, payload: ConnectorSchema): @router.post("/prompts/create", response={200: PromptSchema, 400: DetailSchema}) def create_connector(request, payload: PromptSchema): - # check for admin permissions data = payload.dict() logger.debug(data) try: logger.debug("creating prompt") prompts = Prompt.objects.create(**data) return (200, prompts) - except Exception: - logger.exception("Prompt error") - return (400, {"detail": "The specific prompt cannot be created."}) - + except Exception as ex: + logger.debug(f"prompt not created. Attempting to update.") + # Prompt.objects.filter(name=data['name']) will only return one item as name is unique for every prompt + rows_updated = Prompt.objects.filter(name=data['name']).update(**data) + if rows_updated == 0: + logger.debug(f"nothing to update") + elif rows_updated == 1: + logger.debug(f"prompt updated") + else: + logger.debug(f"something went wrong while creating/updating prompt. message: {ex}") + finally: + if connection.queries: + last_query = connection.queries[-1] + logger.debug(f"last executed SQL: {last_query['sql']}") + @router.post("/packages/create", response={200: PackageSchema, 400: DetailSchema}) def create_connector(request, payload: PackageSchema): diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index b975ac8..c7068e0 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -79,13 +79,13 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "query"] + model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query"] class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "spec", "package_id", "gated", "query"] + model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query"] schemas: List[Dict] diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index a4de0d6..d8bffec 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -4,39 +4,12 @@ "name": "Inventory snapshot", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", "query": "orders_with_product_data", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -46,39 +19,12 @@ "name": "Abandoned customers", "description": "List of users who have abandoned their cart", "query": "select * from {{schema}}.abandon_customers where {{filters}}", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -88,39 +34,12 @@ "name": "Average order value", "description": "Average order value", "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders where {{filters}}", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -130,39 +49,12 @@ "name": "Refunds by date", "description": "List of users who have abandoned their cart", "query": "select * from {{schema}}.abandon_customers where {{filters}}", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -172,39 +64,12 @@ "name": "Sales by category", "description": "Sales by category", "query": "select * from {{schema}}.abandon_customers where {{filters}}", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -214,39 +79,12 @@ "name": "Sales by payment method", "description": "Sales by payment method", "query": "SELECT md5(row(transactions.gateway, COUNT(*)::text, ROUND(SUM(total_price))::text)::text) as id, transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", - "spec": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["timeWindows"], - "properties": { - "timeWindow": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "range": { - "type": "object", - "properties": { - "start": { "type": "string" }, - "end": { "type": "string" } - } - } - } - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { "type": "string" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "value": { "type": "string" }, - "operator": { "type": "string", "enum": ["=", "!="] } - }, - "required": ["label", "name", "type", "value", "operator"] - } - } - } + "filters": [ + {"column": "ORDER_VALUE", "column_type": "integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", diff --git a/init_db/prompt_init.py b/init_db/prompt_init.py index b546678..ebf6caa 100644 --- a/init_db/prompt_init.py +++ b/init_db/prompt_init.py @@ -1,39 +1,33 @@ -""" -Copyright (c) 2024 valmi.io - -Created Date: Wednesday, April 11th 2024, 9:56:52 pm -Author: Rajashekar Varkala @ valmi.io - -""" - import json import logging import os import uuid from os.path import dirname, join - import requests from requests.auth import HTTPBasicAuth logger = logging.getLogger(__name__) +# Load prompt definitions prompt_defs = json.loads(open(join(dirname(__file__), "prompt_def.json"), "r").read()) -for prompt_def in prompt_defs["definitions"]: - logger.debug(prompt_def) +# Function to create or update prompt +def create_or_update_prompt(prompt_def): + # Add a unique ID for prompt creation + prompt_def['id'] = str(uuid.uuid4()) + + # Attempt to create or update the prompt resp = requests.post( f"http://localhost:{os.environ['PORT']}/api/v1/superuser/prompts/create", - json={ - "id": str(uuid.uuid4()), - "name": prompt_def["name"], - "description": prompt_def["description"], - "type": prompt_def["type"], - "query": prompt_def["query"], - "spec": prompt_def["spec"], - "package_id": prompt_def["package_id"], - "gated": prompt_def["gated"], - }, + json=prompt_def, auth=HTTPBasicAuth(os.environ["ADMIN_EMAIL"], os.environ["ADMIN_PASSWORD"]), ) - if resp.status_code != 200: - print("Failed to create prompt. May exist already. Do better - continuing...") + + if resp.status_code == 200: + logger.debug("Prompt created successfully") + return + + +# Iterate through prompt definitions and create or update prompts +for prompt_def in prompt_defs["definitions"]: + create_or_update_prompt(prompt_def) From 796e57ce101ea9a3d9c748c68b1346fc2a508543 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 10 Jun 2024 17:14:09 +0530 Subject: [PATCH 133/159] feat: replaced postgres credential names with VALMI_ENGINE --- core/routes/workspace_api.py | 14 ++++++++++++-- core/services/explore.py | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 11c14db..5470dc2 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import time @@ -319,6 +320,12 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): logger.exception("Sync error") return {"detail": "The specific sync cannot be created."} +# TODO: need to find nice place to do this + + +async def wait_for_run(time: int): + await asyncio.sleep(time) + @router.post("/workspaces/{workspace_id}/syncs/create_with_defaults", response={200: SyncSchema, 500: DetailSchema}) def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): @@ -345,11 +352,11 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): SourceAccessInfo.objects.create(**source_access_info) # creating destination credential destination_credential_payload = CredentialSchemaIn( - name="default warehouse", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + name="VALMI_ENGINE", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) destination_credential = create_credential(request, workspace_id, destination_credential_payload) # creating destination destination_payload = DestinationSchemaIn( - name="default warehouse", credential_id=destination_credential.id, catalog=catalog) + name="VALMI_ENGINE", credential_id=destination_credential.id, catalog=catalog) destination = create_destination(request, workspace_id, destination_payload) data["source"] = Source.objects.get(id=source.id) data["destination"] = Destination.objects.get(id=destination.id) @@ -364,6 +371,9 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data["id"] = uuid.uuid4() logger.debug(data) sync = Sync.objects.create(**data) + asyncio.run(wait_for_run(5)) + payload = SyncStartStopSchemaIn(full_refresh=False) + response = create_new_run(request, workspace_id, sync.id, payload) return sync except Exception: logger.exception("Sync error") diff --git a/core/services/explore.py b/core/services/explore.py index 5c214d0..dac3465 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -83,7 +83,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ credential = {"id": uuid.uuid4()} credential["workspace"] = Workspace.objects.get(id=workspace_id) credential["connector_id"] = "SRC_POSTGRES" - credential["name"] = "SRC_POSTGRES" + credential["name"] = "VALMI_ENGINE" credential["account"] = account credential["status"] = "active" # building query @@ -110,7 +110,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) source = { - "name": "SRC_POSTGRES", + "name": "VALMI_ENGINE", "id": uuid.uuid4() } # creating source object From 23b73d5d6a3b5aecf1776e9fdf073777e100c221 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Mon, 10 Jun 2024 18:18:59 +0530 Subject: [PATCH 134/159] bug: fix spread sheet access check --- core/services/explore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/explore.py b/core/services/explore.py index dac3465..38a8278 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -282,7 +282,7 @@ def is_sheet_accessible(sheet_url: str, workspace_id: str) -> bool: # Check permissions to see if the file is accessible permissions = spreadsheet_metadata.get('permissions', []) for permission in permissions: - if (permission.get('type') == 'user' and permission.get('role') == 'writer') or (permission.get('type') == 'domain' and permission.get('role') == 'writer') or (permission.get('type') == 'anyone' and permission.get('role') == 'writer'): + if (permission.get('type') == 'user' and permission.get('role') == 'owner') or (permission.get('type') == 'domain' and permission.get('role') == 'writer') or (permission.get('type') == 'anyone' and permission.get('role') == 'writer'): return True return False except Exception as e: From cdce44ac31ab5d688f391524024b87c84c2abeda Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Mon, 10 Jun 2024 22:02:18 +0530 Subject: [PATCH 135/159] feat: update Filter schema --- core/schemas/prompt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index c72522b..0dc2a55 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -20,9 +20,9 @@ class TableInfo(Schema): class Filter(Schema): - label: str + column: str + column_type: str operator: str - name: str value: str From e1ff0b259ef26e8bc184d88b8e608e2334cec64f Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Tue, 11 Jun 2024 13:13:56 +0530 Subject: [PATCH 136/159] feat: fix query building from filters --- core/routes/engine_api.py | 2 ++ core/routes/prompt_api.py | 6 ++++++ core/services/prompts.py | 41 +++++++-------------------------------- init_db/prompt_def.json | 4 ++-- 4 files changed, 17 insertions(+), 36 deletions(-) diff --git a/core/routes/engine_api.py b/core/routes/engine_api.py index ef0262a..79cfd5f 100644 --- a/core/routes/engine_api.py +++ b/core/routes/engine_api.py @@ -108,6 +108,8 @@ def create_connector(request, payload: PromptSchema): except Exception as ex: logger.debug(f"prompt not created. Attempting to update.") # Prompt.objects.filter(name=data['name']) will only return one item as name is unique for every prompt + data.pop('id') + logger.debug(f"updating with {data}") rows_updated = Prompt.objects.filter(name=data['name']).update(**data) if rows_updated == 0: logger.debug(f"nothing to update") diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index d08731b..63eb9b4 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -83,6 +83,7 @@ def custom_serializer(obj): def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSchemaIn): try: prompt = Prompt.objects.get(id=prompt_id) + # checking wether prompt is enabled or not if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" @@ -121,3 +122,8 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche except Exception as err: logger.exception(f"preview fetching error:{err}") return (400, {"detail": "Data cannot be fetched."}) + finally: + from django.db import connection + if connection.queries: + last_query = connection.queries[-1] + logger.debug(f"last executed SQL: {last_query['sql']}") diff --git a/core/services/prompts.py b/core/services/prompts.py index 494975f..50f02af 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,10 +1,7 @@ import json import logging -from pathlib import Path -from liquid import Template as LiquidTemplate import requests from decouple import config -from liquid import Environment, FileSystemLoader, Mode, StrictUndefined from core.models import Credential from core.schemas.prompt import (Filter, LastSuccessfulSyncInfo, TableInfo, @@ -21,37 +18,13 @@ def getTemplateFile(cls): @staticmethod def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: - try: - if isinstance(timeWindow, TimeWindow): - timeWindowDict = timeWindow.dict() - else: - timeWindowDict = timeWindow - if isinstance(filters, Filter): - filterList = [filter.__dict__ for filter in filters] - else: - filterList = filters - file_name = PromptService.getTemplateFile() - template_parent_path = Path(__file__).parent.absolute() - env = Environment( - tolerance=Mode.STRICT, - undefined=StrictUndefined, - loader=FileSystemLoader( - f"{str(template_parent_path)}/prompt_templates" - ), - ) - template = env.get_template(file_name) - filters = list(filters) - filterList = [filter.dict() for filter in filters] - rendered_query = template.render(filters=filterList, timeWindow=timeWindowDict) - liquid_template = LiquidTemplate(tableInfo.query) - context = { - "schema": tableInfo.tableSchema, - "filters": rendered_query - } - return liquid_template.render(context) - except Exception as e: - logger.exception(e) - raise e + where_clause = "where 1=1" + for filter in filters: + where_clause += f" and {filter.column} {filter.operator} {filter.value} " + + query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace("{{filters}}", where_clause) + return query + @staticmethod def is_enabled(workspace_id: str, prompt: object) -> bool: diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index d8bffec..52be700 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -18,7 +18,7 @@ { "name": "Abandoned customers", "description": "List of users who have abandoned their cart", - "query": "select * from {{schema}}.abandon_customers where {{filters}}", + "query": "select * from {{schema}}.abandon_customers {{filters}}", "filters": [ {"column": "ORDER_VALUE", "column_type": "integer"} ], @@ -33,7 +33,7 @@ { "name": "Average order value", "description": "Average order value", - "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders where {{filters}}", + "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders {{filters}}", "filters": [ {"column": "ORDER_VALUE", "column_type": "integer"} ], From 352f58133e30aaad65914ad2e4462a619cd243e3 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Tue, 11 Jun 2024 13:28:08 +0530 Subject: [PATCH 137/159] fix: where_clause building for prompt query --- core/services/prompts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/services/prompts.py b/core/services/prompts.py index 50f02af..d5e5eb3 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -20,9 +20,12 @@ def getTemplateFile(cls): def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: where_clause = "where 1=1" for filter in filters: - where_clause += f" and {filter.column} {filter.operator} {filter.value} " - + if filter.column_type in ('integer', 'float'): + where_clause += f" and {filter.column} {filter.operator} {filter.value} " + else: + where_clause += f" and {filter.column} {filter.operator} '{filter.value}' " query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace("{{filters}}", where_clause) + logger.debug(f"prompt query built: {query}") return query From 8a12b6966995995e0bfc83ce84847896b53f3032 Mon Sep 17 00:00:00 2001 From: chaitanya6416 Date: Tue, 11 Jun 2024 15:09:06 +0530 Subject: [PATCH 138/159] feat: add db_column & display_column to promts --- core/services/prompts.py | 12 +++++++----- init_db/prompt_def.json | 12 ++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/core/services/prompts.py b/core/services/prompts.py index d5e5eb3..72f8e9e 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -18,13 +18,15 @@ def getTemplateFile(cls): @staticmethod def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: - where_clause = "where 1=1" - for filter in filters: + where_clause_conditions = " " + for i, filter in enumerate(filters): if filter.column_type in ('integer', 'float'): - where_clause += f" and {filter.column} {filter.operator} {filter.value} " + where_clause_conditions += f" {filter.column} {filter.operator} {filter.value} " else: - where_clause += f" and {filter.column} {filter.operator} '{filter.value}' " - query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace("{{filters}}", where_clause) + where_clause_conditions += f" {filter.column} {filter.operator} '{filter.value}' " + if i != len(filters)-1: + where_clause_conditions += " and " + query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace("{{filters}}", where_clause_conditions) logger.debug(f"prompt query built: {query}") return query diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 52be700..f8ccdcf 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -20,7 +20,7 @@ "description": "List of users who have abandoned their cart", "query": "select * from {{schema}}.abandon_customers {{filters}}", "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} + {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -35,7 +35,7 @@ "description": "Average order value", "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders {{filters}}", "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} + {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -50,7 +50,7 @@ "description": "List of users who have abandoned their cart", "query": "select * from {{schema}}.abandon_customers where {{filters}}", "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} + {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -65,7 +65,7 @@ "description": "Sales by category", "query": "select * from {{schema}}.abandon_customers where {{filters}}", "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} + {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -78,9 +78,9 @@ { "name": "Sales by payment method", "description": "Sales by payment method", - "query": "SELECT md5(row(transactions.gateway, COUNT(*)::text, ROUND(SUM(total_price))::text)::text) as id, transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales FROM {{ schema }}.orders as orders join {{ schema }}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", + "query": "SELECT md5(row(transactions.gateway, COUNT(*)::text, ROUND(SUM(total_price))::text)::text) as id, transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales FROM {{schema}}.orders as orders join {{schema}}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} + {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], From 2fb4f0f82568ce3f2a56a192c531fbad08013baf Mon Sep 17 00:00:00 2001 From: Nagendra Date: Wed, 12 Jun 2024 12:08:15 +0530 Subject: [PATCH 139/159] feat: add Google OAuth 2.0 credentials to .env example --- .env-example | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.env-example b/.env-example index 551713a..6f65268 100644 --- a/.env-example +++ b/.env-example @@ -46,4 +46,8 @@ OTEL_EXPORTER_OTLP_INSECURE=True DATA_WAREHOUSE_URL="********************" DATA_WAREHOUSE_USERNAME="****************" DATA_WAREHOUSE_PASSWORD="***************8" -DATA_WAREHOUSE_DB_NAME="******************" \ No newline at end of file +DATA_WAREHOUSE_DB_NAME="******************" + + +NEXTAUTH_GOOGLE_CLIENT_ID="*********************" +NEXTAUTH_GOOGLE_CLIENT_SECRET="****************" \ No newline at end of file From c3b23da84c12f45d2cf6240936bd1dadde4813ad Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 14 Jun 2024 12:17:00 +0530 Subject: [PATCH 140/159] feat: checking run manager status for sync status --- core/routes/explore_api.py | 14 ++++++------ core/routes/prompt_api.py | 6 +++--- core/routes/workspace_api.py | 6 +++--- core/services/explore.py | 10 ++++----- core/services/prompts.py | 7 +++--- core/services/source_catalog.json | 2 +- init_db/prompt_def.json | 36 +++++++++++++++++++++++++------ 7 files changed, 54 insertions(+), 27 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index e3505db..1ed4be5 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -10,7 +10,6 @@ from core.models import Account, Explore, Prompt, Workspace from core.services.explore import ExploreService -from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -32,6 +31,7 @@ def get_explores(request, workspace_id): explore.sync_id = str(explore.sync.id) latest_sync_info = ExploreService.get_latest_sync_info(explore.sync.id) logger.debug(latest_sync_info) + # as we are using full_refresh as true in explore creation enable only if previous sync got succeded # checking whether run is created for explore or not if latest_sync_info.found == False: explore.enabled = False @@ -46,11 +46,13 @@ def get_explores(request, workspace_id): else: explore.last_sync_result = latest_sync_info.status.upper() explore.sync_state = 'IDLE' + if latest_sync_info.status == 'success': + explore.last_sync_succeeded_at = latest_sync_info.created_at explore.last_sync_created_at = latest_sync_info.created_at # adding last successful sync info - last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) - if last_successful_sync_info.found == True: - explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at + # last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) + # if last_successful_sync_info.found == True: + # explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at return explores except Exception: logger.exception("explores listing error") @@ -97,8 +99,8 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): spreadsheet_title, data["name"], data["sheet_url"], workspace_id, account) spreadsheet_url = destination_data[0] destination = destination_data[1] - # create sync - sync = ExploreService.create_sync(source, destination, workspace_id) + # creating the sync + sync = ExploreService.create_sync(data["name"], source, destination, workspace_id) # creating explore del data["schema_id"] del data["filters"] diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 63eb9b4..ef07ef7 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -83,7 +83,7 @@ def custom_serializer(obj): def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSchemaIn): try: prompt = Prompt.objects.get(id=prompt_id) - + # checking wether prompt is enabled or not if not PromptService.is_enabled(workspace_id, prompt): detail_message = f"The prompt is not enabled. Please add '{prompt.type}' connector" @@ -93,8 +93,8 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche sync_id = sync.id # checking wether sync has finished or not(from shopify to DB) latest_sync_info = PromptService.is_sync_finished(sync_id) - # if latest_sync_info.found == False or latest_sync_info.status == 'running': - # return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} + if latest_sync_info.found == False: + return 400, {"detail": "The sync is not finished. Please wait for the sync to finish."} storage_credentials = StorageCredentials.objects.get(id=prompt_req.schema_id) schema_name = storage_credentials.connector_config["schema"] table_info = TableInfo( diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 5470dc2..1e95b26 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -309,7 +309,7 @@ def create_sync(request, workspace_id, payload: SyncSchemaIn): del data["destination_id"] schedule = {} if len(data["schedule"]) == 0: - schedule["run_interval"] = 3600000 + schedule["run_interval"] = 86400000 data["schedule"] = schedule data["workspace"] = Workspace.objects.get(id=workspace_id) data["id"] = uuid.uuid4() @@ -352,11 +352,11 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): SourceAccessInfo.objects.create(**source_access_info) # creating destination credential destination_credential_payload = CredentialSchemaIn( - name="VALMI_ENGINE", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) + name="VALMI_DATA_STORE", account=data["account"], connector_type="DEST_POSTGRES-DEST", connector_config=storage_credentials.connector_config) destination_credential = create_credential(request, workspace_id, destination_credential_payload) # creating destination destination_payload = DestinationSchemaIn( - name="VALMI_ENGINE", credential_id=destination_credential.id, catalog=catalog) + name="VALMI_DATA_STORE", credential_id=destination_credential.id, catalog=catalog) destination = create_destination(request, workspace_id, destination_payload) data["source"] = Source.objects.get(id=source.id) data["destination"] = Destination.objects.get(id=destination.id) diff --git a/core/services/explore.py b/core/services/explore.py index 38a8278..5e5955d 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -83,7 +83,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ credential = {"id": uuid.uuid4()} credential["workspace"] = Workspace.objects.get(id=workspace_id) credential["connector_id"] = "SRC_POSTGRES" - credential["name"] = "VALMI_ENGINE" + credential["name"] = "VALMI_DATA_STORE" credential["account"] = account credential["status"] = "active" # building query @@ -110,7 +110,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ credential["connector_config"] = connector_config cred = Credential.objects.create(**credential) source = { - "name": "VALMI_ENGINE", + "name": "VALMI_DATA_STORE", "id": uuid.uuid4() } # creating source object @@ -190,18 +190,18 @@ def create_destination(spreadsheet_title: str, spreadsheet_name: str, sheet_url: raise Exception("unable to create destination") @staticmethod - def create_sync(source: object, destination: object, workspace_id: str) -> object: + def create_sync(name: str, source: object, destination: object, workspace_id: str) -> object: try: logger.debug("creating sync in service") logger.debug(source.id) sync_config = { - "name": "Warehouse to sheets", + "name": name, "id": uuid.uuid4(), "status": "active", "ui_state": {} } - schedule = {"run_interval": 3600000} + schedule = {"run_interval": 86400000} sync_config["schedule"] = schedule sync_config["source"] = source sync_config["destination"] = destination diff --git a/core/services/prompts.py b/core/services/prompts.py index 72f8e9e..2d04379 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -24,13 +24,13 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) - where_clause_conditions += f" {filter.column} {filter.operator} {filter.value} " else: where_clause_conditions += f" {filter.column} {filter.operator} '{filter.value}' " - if i != len(filters)-1: + if i != len(filters) - 1: where_clause_conditions += " and " - query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace("{{filters}}", where_clause_conditions) + query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace( + "{{filters}}", where_clause_conditions) logger.debug(f"prompt query built: {query}") return query - @staticmethod def is_enabled(workspace_id: str, prompt: object) -> bool: connector_ids = list(Credential.objects.filter(workspace_id=workspace_id).values('connector_id').distinct()) @@ -43,6 +43,7 @@ def is_enabled(workspace_id: str, prompt: object) -> bool: def is_sync_finished(sync_id: str) -> LastSuccessfulSyncInfo: try: response = requests.get(f"{ACTIVATION_URL}/syncs/{sync_id}/last_successful_sync") + logger.debug(response) json_string = response.content.decode('utf-8') logger.debug(json_string) last_success_sync_dict = json.loads(json_string) diff --git a/core/services/source_catalog.json b/core/services/source_catalog.json index 722d342..3df6484 100644 --- a/core/services/source_catalog.json +++ b/core/services/source_catalog.json @@ -4,7 +4,7 @@ "id_key": "id", "stream": { }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "append" } ] diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index f8ccdcf..8c34fee 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -3,7 +3,7 @@ { "name": "Inventory snapshot", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "query": "orders_with_product_data", + "query": "inventory_snapshot.sql", "filters": [ {"column": "ORDER_VALUE", "column_type": "integer"} ], @@ -18,7 +18,7 @@ { "name": "Abandoned customers", "description": "List of users who have abandoned their cart", - "query": "select * from {{schema}}.abandon_customers {{filters}}", + "query": "inventory_snapshot.sql", "filters": [ {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], @@ -33,7 +33,7 @@ { "name": "Average order value", "description": "Average order value", - "query": "select avg(total_price)::NUMERIC(10,2) from {{schema}}.orders {{filters}}", + "query": "inventory_snapshot.sql", "filters": [ {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], @@ -48,7 +48,7 @@ { "name": "Refunds by date", "description": "List of users who have abandoned their cart", - "query": "select * from {{schema}}.abandon_customers where {{filters}}", + "query": "inventory_snapshot.sql", "filters": [ {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], @@ -63,7 +63,7 @@ { "name": "Sales by category", "description": "Sales by category", - "query": "select * from {{schema}}.abandon_customers where {{filters}}", + "query": "inventory_snapshot.sql", "filters": [ {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], @@ -78,7 +78,7 @@ { "name": "Sales by payment method", "description": "Sales by payment method", - "query": "SELECT md5(row(transactions.gateway, COUNT(*)::text, ROUND(SUM(total_price))::text)::text) as id, transactions.gateway AS Payment_Method, COUNT(*) AS Total_Orders, ROUND(SUM(total_price)) as total_sales FROM {{schema}}.orders as orders join {{schema}}.transactions as transactions on transactions.order_id = orders.id where transactions.status='SUCCESS' and transactions.kind = 'SALE' and {{filters}} GROUP BY Payment_Method ORDER BY Total_Sales DESC", + "query": "inventory_snapshot.sql", "filters": [ {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], @@ -89,6 +89,30 @@ "type": "SRC_SHOPIFY", "package_id": "P0", "gated": true + }, + { + "name": "Customer Segmentation Builder", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "query": "select * from {{schema}}.customer_segmentation where {{filters}}", + "filters": [ + {"display_column": "email", "db_column": "customer_email", "column_type": "string"}, + {"display_column":"phone","db_column":"customer_phone","column_type":"string"}, + {"display_column":"city","db_column":"city","column_type":"string"}, + {"display_column":"state","db_column":"state'","column_type":"string"}, + {"display_column":"country","db_column":"country","column_type":"string"}, + {"display_column":"zip","db_column":"zip","column_type":"string"}, + {"display_column":"total_sales","db_column":"total_sales","column_type":"integer"}, + {"display_column":"total_orders","db_column":"total_orders","column_type":"integer"}, + {"display_column":"number_of_cancellations","db_column":"number_of_cancellations","column_type":"integer"}, + {"display_column":"AOV","db_column":"AOV","column_type":"integer"} + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true } ] } From 96680afa2fc455ad6d901aa36c20adb0f0e55ffa Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 14 Jun 2024 16:40:14 +0530 Subject: [PATCH 141/159] feat: Added time window in prompt --- core/services/prompt_templates/prompts.liquid | 2 +- core/services/prompts.py | 44 ++++++++++++++----- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 22a5a13..9c3272b 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -19,4 +19,4 @@ {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} {%- else -%} {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' -{%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/core/services/prompts.py b/core/services/prompts.py index 2d04379..e6d73ed 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -1,7 +1,10 @@ import json import logging +from pathlib import Path +from liquid import Template as LiquidTemplate import requests from decouple import config +from liquid import Environment, FileSystemLoader, Mode, StrictUndefined from core.models import Credential from core.schemas.prompt import (Filter, LastSuccessfulSyncInfo, TableInfo, @@ -18,18 +21,37 @@ def getTemplateFile(cls): @staticmethod def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: - where_clause_conditions = " " - for i, filter in enumerate(filters): - if filter.column_type in ('integer', 'float'): - where_clause_conditions += f" {filter.column} {filter.operator} {filter.value} " + try: + if isinstance(timeWindow, TimeWindow): + timeWindowDict = timeWindow.dict() + else: + timeWindowDict = timeWindow + if isinstance(filters, Filter): + filterList = [filter.__dict__ for filter in filters] else: - where_clause_conditions += f" {filter.column} {filter.operator} '{filter.value}' " - if i != len(filters) - 1: - where_clause_conditions += " and " - query = tableInfo.query.replace("{{schema}}", tableInfo.tableSchema).replace( - "{{filters}}", where_clause_conditions) - logger.debug(f"prompt query built: {query}") - return query + filterList = filters + file_name = PromptService.getTemplateFile() + template_parent_path = Path(__file__).parent.absolute() + env = Environment( + tolerance=Mode.STRICT, + undefined=StrictUndefined, + loader=FileSystemLoader( + f"{str(template_parent_path)}/prompt_templates" + ), + ) + template = env.get_template(file_name) + filters = list(filters) + filterList = [filter.dict() for filter in filters] + rendered_query = template.render(filters=filterList, timeWindow=timeWindowDict) + liquid_template = LiquidTemplate(tableInfo.query) + context = { + "schema": tableInfo.tableSchema, + "filters": rendered_query + } + return liquid_template.render(context) + except Exception as e: + logger.exception(e) + raise e @staticmethod def is_enabled(workspace_id: str, prompt: object) -> bool: From 6a31c3a54a17e1c47225e9a7b1838aca557abaa5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Fri, 14 Jun 2024 18:48:00 +0530 Subject: [PATCH 142/159] feat: added prompt def for new prompts --- core/services/prompt_templates/prompts.liquid | 4 ++-- core/services/prompts.py | 7 ++++++- init_db/prompt_def.json | 15 ++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 9c3272b..bcaf3ea 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -1,5 +1,5 @@ {%- assign filterStr = "" -%} -{% for filter in filters %} +{%- for filter in filters -%} {% capture name %} {{ filter.name }} {% endcapture %} @@ -11,7 +11,7 @@ {% if forloop.last == false %} {% assign filterStr = filterStr | append: " and " %} {% endif %} -{% endfor %} +{%- endfor -%} {%- if filterStr != "" -%} {% assign filterStr = filterStr | append: " and " %} {%- endif -%} diff --git a/core/services/prompts.py b/core/services/prompts.py index e6d73ed..0cadfa3 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -48,7 +48,12 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) - "schema": tableInfo.tableSchema, "filters": rendered_query } - return liquid_template.render(context) + query = liquid_template.render(context) + logger.debug(type(query)) + query = query.replace("\\", "") + logger.debug(query) + logger.debug(query) + return query except Exception as e: logger.exception(e) raise e diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 8c34fee..529d1ee 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -46,15 +46,12 @@ "gated": true }, { - "name": "Refunds by date", - "description": "List of users who have abandoned their cart", - "query": "inventory_snapshot.sql", + "name": "Refunds on a daily basis", + "description": "Get the list of refunds on any day, filtered by the date", + "query": " select md5(row(DATE(updated_at),SUM(amount :: numeric))::text) as id,DATE(o.updated_at) as date, SUM(amount :: numeric) AS total_refund from {{schema}}.orders as o join {{schema}}.orders_refunds_transactions as ort on o.id = ort.order_id where {{filters}} group by DATE(o.updated_at)", "filters": [ - {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} ], "operators": { - "string": ["=", "!=", "IN", "NOT IN"], - "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", @@ -77,10 +74,10 @@ }, { "name": "Sales by payment method", - "description": "Sales by payment method", - "query": "inventory_snapshot.sql", + "description": "Get the data of overall sales by different payment methods", + "query": "select Payment_Method, count(*) as Total_Orders, round(sum(total_price)) as Total_Sales from sales_by_payment_method where {{filters}} group by Payment_Method", "filters": [ - {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} + {"display_column": "payment_method", "db_column": "payment_method", "column_type": "string"} ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], From ae2b91593edb40f9c26da9b03020a62ef5c3b5a9 Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Mon, 17 Jun 2024 18:00:48 +0530 Subject: [PATCH 143/159] support for time grain --- .../0033_prompt_time_grain_enabled.py | 18 ++++++++++ core/models.py | 1 + core/routes/engine_api.py | 35 ++++++++----------- core/routes/explore_api.py | 2 +- core/routes/prompt_api.py | 8 +++-- core/schemas/explore.py | 3 +- core/schemas/prompt.py | 10 ++++++ core/schemas/schemas.py | 7 ++-- core/services/explore.py | 9 ++--- core/services/prompts.py | 5 +-- 10 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 core/migrations/0033_prompt_time_grain_enabled.py diff --git a/core/migrations/0033_prompt_time_grain_enabled.py b/core/migrations/0033_prompt_time_grain_enabled.py new file mode 100644 index 0000000..b17aefc --- /dev/null +++ b/core/migrations/0033_prompt_time_grain_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-06-17 11:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0032_auto_20240607_1146'), + ] + + operations = [ + migrations.AddField( + model_name='prompt', + name='time_grain_enabled', + field=models.BooleanField(default=False), + ), + ] diff --git a/core/models.py b/core/models.py index 0499eaf..d948d33 100644 --- a/core/models.py +++ b/core/models.py @@ -153,6 +153,7 @@ class Prompt(models.Model): query = models.CharField(max_length=1000,null=False, blank=False,default="query") package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") gated = models.BooleanField(null=False, blank = False, default=True) + time_grain_enabled = models.BooleanField(null=False, blank=False, default=False) class SourceAccessInfo(models.Model): source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info",primary_key=True) diff --git a/core/routes/engine_api.py b/core/routes/engine_api.py index 79cfd5f..8226381 100644 --- a/core/routes/engine_api.py +++ b/core/routes/engine_api.py @@ -6,28 +6,21 @@ """ +import json import logging from typing import Dict, List from decouple import Csv, config +# from opentelemetry import trace from ninja import Router - -from core.schemas.schemas import ConnectorSchema, DetailSchema, PackageSchema, PromptSchema, SyncSchema - -from ..models import ( - Connector, - Package, - Prompt, - Sync, - OAuthApiKeys -) -import json - from opentelemetry.metrics import get_meter_provider -# from opentelemetry import trace -from django.db import connection + +from core.schemas.schemas import (ConnectorSchema, DetailSchema, PackageSchema, + PromptSchema, SyncSchema) from valmi_app_backend.utils import replace_values_in_json +from ..models import Connector, OAuthApiKeys, Package, Prompt, Sync + router = Router() # Get an instance of a logger @@ -103,8 +96,8 @@ def create_connector(request, payload: PromptSchema): logger.debug(data) try: logger.debug("creating prompt") - prompts = Prompt.objects.create(**data) - return (200, prompts) + prompt = Prompt.objects.create(**data) + return (200, prompt) except Exception as ex: logger.debug(f"prompt not created. Attempting to update.") # Prompt.objects.filter(name=data['name']) will only return one item as name is unique for every prompt @@ -116,11 +109,11 @@ def create_connector(request, payload: PromptSchema): elif rows_updated == 1: logger.debug(f"prompt updated") else: - logger.debug(f"something went wrong while creating/updating prompt. message: {ex}") - finally: - if connection.queries: - last_query = connection.queries[-1] - logger.debug(f"last executed SQL: {last_query['sql']}") + msg = f"something went wrong while creating/updating prompt. message: {ex}" + logger.debug(msg) + return (400, {"detail": msg}) + return (200, data) + @router.post("/packages/create", response={200: PackageSchema, 400: DetailSchema}) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 1ed4be5..9209466 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -92,7 +92,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["account"] = account # create source source = ExploreService.create_source( - table_name, data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], workspace_id, account) + table_name, data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], data["time_grain"], workspace_id, account) # create destination spreadsheet_title = f"valmi.io {prompt.name} sheet" destination_data = ExploreService.create_destination( diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index ef07ef7..5ec89fb 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -1,7 +1,7 @@ import datetime -from decimal import Decimal import json import logging +from decimal import Decimal from typing import List import psycopg2 @@ -10,7 +10,7 @@ from core.models import (Credential, Prompt, Source, SourceAccessInfo, StorageCredentials, Sync) -from core.schemas.prompt import PromptPreviewSchemaIn, TableInfo +from core.schemas.prompt import PromptPreviewSchemaIn, TableInfo, TimeGrain from core.schemas.schemas import (DetailSchema, PromptByIdSchema, PromptSchemaOut) from core.services.prompts import PromptService @@ -65,6 +65,8 @@ def get_prompt_by_id(request, workspace_id, prompt_id): # Convert schemas dictionary to a list (optional) final_schemas = list(schemas.values()) prompt.schemas = final_schemas + if prompt.time_grain_enabled: + prompt.time_grain = TimeGrain.members() logger.debug(prompt) return prompt except Exception: @@ -102,7 +104,7 @@ def preview_data(request, workspace_id, prompt_id, prompt_req: PromptPreviewSche query=prompt.query ) - query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters) + query = PromptService().build(table_info, prompt_req.time_window, prompt_req.filters, prompt_req.time_grain) query = query + " limit 10" logger.debug(query) host = storage_credentials.connector_config.get('host') diff --git a/core/schemas/explore.py b/core/schemas/explore.py index ca30363..1ff57dd 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -2,7 +2,7 @@ from typing import Dict, Optional from ninja import Field, ModelSchema, Schema from core.models import Explore -from core.schemas.prompt import Filter, TimeWindow +from core.schemas.prompt import Filter, TimeWindow, TimeGrain from core.schemas.schemas import AccountSchema, CamelSchemaConfig, PromptSchema, WorkspaceSchema @@ -43,6 +43,7 @@ class ExploreSchemaIn(Schema): schema_id: str time_window: TimeWindow filters: list[Filter] + time_grain: Optional[TimeGrain] class LatestSyncInfo(Schema): diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index 0dc2a55..7e18c81 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -1,4 +1,5 @@ from datetime import datetime +from enum import Enum from typing import Optional from ninja import Schema @@ -13,6 +14,13 @@ class TimeWindow(Schema): label: str range: TimeWindowRange +class TimeGrain(str, Enum): + day = 'day' + week = 'week' + month = 'month' + def members(): + return TimeGrain._member_names_ + class TableInfo(Schema): tableSchema: str @@ -29,9 +37,11 @@ class Filter(Schema): class PromptPreviewSchemaIn(Schema): schema_id: str time_window: TimeWindow + time_grain: Optional[TimeGrain] filters: list[Filter] class LastSuccessfulSyncInfo(Schema): found: bool run_end_at: Optional[datetime] = None + run_end_at: Optional[datetime] = None diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index c7068e0..86fd685 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -16,6 +16,7 @@ from core.models import (Account, Connector, Credential, Destination, OAuthApiKeys, Organization, Package, Prompt, Source, Sync, Workspace) +from core.schemas.prompt import TimeGrain def camel_to_snake(s): @@ -79,13 +80,14 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query"] + model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query", "time_grain_enabled"] class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query"] + model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query", "time_grain_enabled"] + time_grain: List[TimeGrain] = None schemas: List[Dict] @@ -289,3 +291,4 @@ class SocialUser(Schema): class SocialAuthLoginSchema(Schema): account: SocialAccount user: SocialUser + user: SocialUser diff --git a/core/services/explore.py b/core/services/explore.py index 5e5955d..003c5d4 100644 --- a/core/services/explore.py +++ b/core/services/explore.py @@ -16,7 +16,7 @@ Prompt, Source, StorageCredentials, Sync, Workspace) from core.routes.workspace_api import create_new_run from core.schemas.explore import LatestSyncInfo -from core.schemas.prompt import Filter, TableInfo, TimeWindow +from core.schemas.prompt import Filter, TableInfo, TimeGrain, TimeWindow from core.services.prompts import PromptService logger = logging.getLogger(__name__) @@ -77,7 +77,7 @@ def create_spreadsheet(title: str, name: str, refresh_token: str) -> str: raise Exception("spreadhseet creation failed") @staticmethod - def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_window: TimeWindow, filters: list[Filter], workspace_id: str, account: object) -> object: + def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_window: TimeWindow, filters: list[Filter], time_grain: TimeGrain, workspace_id: str, account: object) -> object: try: # creating source credentail credential = {"id": uuid.uuid4()} @@ -95,7 +95,7 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ tableSchema=namespace, query=prompt.query ) - query = PromptService().build(table_info, time_window, filters) + query = PromptService().build(table_info, time_window, filters, time_grain) # creating source credentials connector_config = { "ssl": storage_credential.connector_config["ssl"], @@ -129,10 +129,11 @@ def create_source(explore_table_name: str, prompt_id: str, schema_id: str, time_ json_file_path = join(dirname(__file__), 'source_catalog.json') with open(json_file_path, 'r') as openfile: source_catalog = json.load(openfile) + database = connector_config["database"] source_catalog["streams"][0]["stream"] = response_json["catalog"]["streams"][0] source_catalog["streams"][0]["stream"][ "name" - ] = explore_table_name + ] = f"{database}.{namespace}.{explore_table_name}" source["catalog"] = source_catalog source["status"] = "active" logger.debug(source_catalog) diff --git a/core/services/prompts.py b/core/services/prompts.py index 0cadfa3..4a07efb 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -20,7 +20,7 @@ def getTemplateFile(cls): return 'prompts.liquid' @staticmethod - def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) -> str: + def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], time_grain: str='day') -> str: try: if isinstance(timeWindow, TimeWindow): timeWindowDict = timeWindow.dict() @@ -46,7 +46,8 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter]) - liquid_template = LiquidTemplate(tableInfo.query) context = { "schema": tableInfo.tableSchema, - "filters": rendered_query + "filters": rendered_query, + "timegrain": time_grain } query = liquid_template.render(context) logger.debug(type(query)) From f0275b99ef762c47909545a02754a6bc8bd957aa Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Mon, 17 Jun 2024 18:01:02 +0530 Subject: [PATCH 144/159] support for time grain --- init_db/prompt_def.json | 99 +++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 529d1ee..77d3713 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -4,9 +4,7 @@ "name": "Inventory snapshot", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", "query": "inventory_snapshot.sql", - "filters": [ - {"column": "ORDER_VALUE", "column_type": "integer"} - ], + "filters": [{ "column": "ORDER_VALUE", "column_type": "integer" }], "operators": { "string": ["=", "!=", "IN", "NOT IN"], "integer": ["=", ">", "<", ">=", "<=", "!="] @@ -20,7 +18,11 @@ "description": "List of users who have abandoned their cart", "query": "inventory_snapshot.sql", "filters": [ - {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} + { + "display_column": "payment_method", + "db_column": "transactions.gateway", + "column_type": "string" + } ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -35,7 +37,11 @@ "description": "Average order value", "query": "inventory_snapshot.sql", "filters": [ - {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} + { + "display_column": "payment_method", + "db_column": "transactions.gateway", + "column_type": "string" + } ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -48,21 +54,24 @@ { "name": "Refunds on a daily basis", "description": "Get the list of refunds on any day, filtered by the date", - "query": " select md5(row(DATE(updated_at),SUM(amount :: numeric))::text) as id,DATE(o.updated_at) as date, SUM(amount :: numeric) AS total_refund from {{schema}}.orders as o join {{schema}}.orders_refunds_transactions as ort on o.id = ort.order_id where {{filters}} group by DATE(o.updated_at)", - "filters": [ - ], - "operators": { - }, + "query": "select md5(row(DATE_TRUNC('{{timegrain}}',o.updated_at),SUM(amount :: numeric))::text) as id,DATE_TRUNC('{{timegrain}}',o.updated_at) as date, SUM(amount :: numeric) AS total_refund from {{schema}}.orders as o join {{schema}}.orders_refunds_transactions as ort on o.id = ort.order_id where {{filters}} group by DATE_TRUNC('{{timegrain}}',o.updated_at) order by DATE_TRUNC('{{timegrain}}',o.updated_at)", + "filters": [], + "operators": {}, "type": "SRC_SHOPIFY", "package_id": "P0", - "gated": true + "gated": true, + "time_grain_enabled": true }, { "name": "Sales by category", "description": "Sales by category", "query": "inventory_snapshot.sql", "filters": [ - {"display_column": "payment_method", "db_column": "transactions.gateway", "column_type": "string"} + { + "display_column": "payment_method", + "db_column": "transactions.gateway", + "column_type": "string" + } ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -77,7 +86,11 @@ "description": "Get the data of overall sales by different payment methods", "query": "select Payment_Method, count(*) as Total_Orders, round(sum(total_price)) as Total_Sales from sales_by_payment_method where {{filters}} group by Payment_Method", "filters": [ - {"display_column": "payment_method", "db_column": "payment_method", "column_type": "string"} + { + "display_column": "payment_method", + "db_column": "payment_method", + "column_type": "string" + } ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], @@ -92,16 +105,56 @@ "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", "query": "select * from {{schema}}.customer_segmentation where {{filters}}", "filters": [ - {"display_column": "email", "db_column": "customer_email", "column_type": "string"}, - {"display_column":"phone","db_column":"customer_phone","column_type":"string"}, - {"display_column":"city","db_column":"city","column_type":"string"}, - {"display_column":"state","db_column":"state'","column_type":"string"}, - {"display_column":"country","db_column":"country","column_type":"string"}, - {"display_column":"zip","db_column":"zip","column_type":"string"}, - {"display_column":"total_sales","db_column":"total_sales","column_type":"integer"}, - {"display_column":"total_orders","db_column":"total_orders","column_type":"integer"}, - {"display_column":"number_of_cancellations","db_column":"number_of_cancellations","column_type":"integer"}, - {"display_column":"AOV","db_column":"AOV","column_type":"integer"} + { + "display_column": "email", + "db_column": "customer_email", + "column_type": "string" + }, + { + "display_column": "phone", + "db_column": "customer_phone", + "column_type": "string" + }, + { + "display_column": "city", + "db_column": "city", + "column_type": "string" + }, + { + "display_column": "state", + "db_column": "state'", + "column_type": "string" + }, + { + "display_column": "country", + "db_column": "country", + "column_type": "string" + }, + { + "display_column": "zip", + "db_column": "zip", + "column_type": "string" + }, + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" + }, + { + "display_column": "total_orders", + "db_column": "total_orders", + "column_type": "integer" + }, + { + "display_column": "number_of_cancellations", + "db_column": "number_of_cancellations", + "column_type": "integer" + }, + { + "display_column": "AOV", + "db_column": "AOV", + "column_type": "integer" + } ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], From 33d5b64e1c838e512e89f0b5f1f0c489a96dfcd0 Mon Sep 17 00:00:00 2001 From: chaitanya6416 <36512605+chaitanya6416@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:21:52 +0530 Subject: [PATCH 145/159] Update core/services/prompt_templates/prompts.liquid --- core/services/prompt_templates/prompts.liquid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index bcaf3ea..3d3fbad 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -1,7 +1,7 @@ {%- assign filterStr = "" -%} {%- for filter in filters -%} {% capture name %} - {{ filter.name }} + {{ filter.column }} {% endcapture %} {% capture value %} '{{ filter.value }}' From 13ceeadbb50733583a1c30e820a0fd1f8c38d035 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 11:28:49 +0530 Subject: [PATCH 146/159] feat: added filters for prompts --- core/routes/explore_api.py | 2 + core/routes/prompt_api.py | 2 + core/routes/workspace_api.py | 2 +- core/services/prompt_templates/prompts.liquid | 6 +- core/services/prompts.py | 5 +- init_db/prompt_def.json | 113 +++++++++++++++++- 6 files changed, 117 insertions(+), 13 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 9209466..578b86a 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -101,11 +101,13 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): destination = destination_data[1] # creating the sync sync = ExploreService.create_sync(data["name"], source, destination, workspace_id) + logger.debug("After sync") # creating explore del data["schema_id"] del data["filters"] del data["time_window"] del data["sheet_url"] + del data["time_grain"] # data["name"] = f"valmiio {prompt.name}" data["sync"] = sync data["ready"] = False diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index 5ec89fb..e54a278 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -77,6 +77,8 @@ def get_prompt_by_id(request, workspace_id, prompt_id): def custom_serializer(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() + if isinstance(obj, datetime.date): + return obj.isoformat() if isinstance(obj, Decimal): return str(obj) diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index 1e95b26..a93b614 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -362,7 +362,7 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): data["destination"] = Destination.objects.get(id=destination.id) del data["account"] if data["schedule"] is None: - schedule = {"run_interval": 3600000} + schedule = {"run_interval": 86400000} data["schedule"] = schedule data["workspace"] = Workspace.objects.get(id=workspace_id) if data["ui_state"] is None: diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index bcaf3ea..313d618 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -1,13 +1,13 @@ {%- assign filterStr = "" -%} {%- for filter in filters -%} - {% capture name %} - {{ filter.name }} + {% capture column %} + {{ filter.column }} {% endcapture %} {% capture value %} '{{ filter.value }}' {% endcapture %} {% assign operator = filter.operator %} - {% assign filterStr = filterStr | append: name | append: " " | append: operator | append: " " | append: value %} + {% assign filterStr = filterStr | append: column | append: " " | append: operator | append: " " | append: value %} {% if forloop.last == false %} {% assign filterStr = filterStr | append: " and " %} {% endif %} diff --git a/core/services/prompts.py b/core/services/prompts.py index 4a07efb..32da54c 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -20,7 +20,7 @@ def getTemplateFile(cls): return 'prompts.liquid' @staticmethod - def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], time_grain: str='day') -> str: + def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], time_grain: str = 'day') -> str: try: if isinstance(timeWindow, TimeWindow): timeWindowDict = timeWindow.dict() @@ -50,9 +50,6 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], t "timegrain": time_grain } query = liquid_template.render(context) - logger.debug(type(query)) - query = query.replace("\\", "") - logger.debug(query) logger.debug(query) return query except Exception as e: diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 77d3713..2e14e7a 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -3,8 +3,40 @@ { "name": "Inventory snapshot", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", - "query": "inventory_snapshot.sql", - "filters": [{ "column": "ORDER_VALUE", "column_type": "integer" }], + "query": "select * from {{schema}}.inventory_snapshot where {{filters}}", + "filters": [ + { + "display_column": "product_title", + "db_column": "product_title", + "column_type": "string" + }, + { + "display_column": "variant_title", + "db_column": "variant_title", + "column_type": "string" + }, + { + "display_column": "product_type", + "db_column": "product_type", + "column_type": "string" + }, + { + "display_column": "sku", + "db_column": "sku", + "column_type": "string" + }, + { + "display_column": "price", + "db_column": "price", + "column_type": "string" + }, + { + "display_column": "cost", + "db_column": "cost", + "column_type": "string" + } + + ], "operators": { "string": ["=", "!=", "IN", "NOT IN"], "integer": ["=", ">", "<", ">=", "<=", "!="] @@ -16,12 +48,81 @@ { "name": "Abandoned customers", "description": "List of users who have abandoned their cart", - "query": "inventory_snapshot.sql", + "query": "select * from {{schema}}.abandon_customers where {{filters}}", "filters": [ { - "display_column": "payment_method", - "db_column": "transactions.gateway", + "display_column": "name", + "db_column": "name", "column_type": "string" + }, + { + "display_column": "email", + "db_column": "email", + "column_type": "string" + }, + { + "display_column": "phone", + "db_column": "phone", + "column_type": "string" + }, + { + "display_column": "cart_value", + "db_column": "cart_value", + "column_type": "integer" + }, + { + "display_column": "no_of_items_in_cart", + "db_column": "no_of_items_in_cart", + "column_type": "integer" + } + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "At risk customers", + "description": "Know your customers who haven’t bought anything in the recent past and need a reminder or two", + "query": "select * from {{schema}}.at_risk_customers where {{filters}}", + "filters": [ + { + "display_column": "first_name", + "db_column": "first_name", + "column_type": "string" + }, + { + "display_column": "last_name", + "db_column": "last_name", + "column_type": "string" + }, + { + "display_column": "phone_number", + "db_column": "phone_number", + "column_type": "string" + }, + { + "display_column": "total_orders", + "db_column": "total_orders", + "column_type": "integer" + }, + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" + }, + { + "display_column": "email", + "db_column": "email", + "column_type": "string" + }, + { + "display_column": "average_order_value", + "db_column": "average_order_value", + "column_type": "integer" } ], "operators": { @@ -32,6 +133,7 @@ "package_id": "P0", "gated": true }, + { "name": "Average order value", "description": "Average order value", @@ -51,6 +153,7 @@ "package_id": "P0", "gated": true }, + { "name": "Refunds on a daily basis", "description": "Get the list of refunds on any day, filtered by the date", From 6f177a5ff6fd6c9c00119d3cbb5bf0715802c81b Mon Sep 17 00:00:00 2001 From: gane5hvarma Date: Tue, 18 Jun 2024 12:16:21 +0530 Subject: [PATCH 147/159] support for time grain --- core/routes/explore_api.py | 20 ++++++++++++-------- core/schemas/explore.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 578b86a..6354cc0 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -1,16 +1,17 @@ import asyncio import logging -from typing import List import uuid +from typing import List + from decouple import config -from core.models import Account, Explore, Prompt, Workspace -from core.schemas.schemas import DetailSchema, SyncStartStopSchemaIn -from core.schemas.explore import ExploreSchema, ExploreSchemaIn, ExploreSchemaOut from ninja import Router from core.models import Account, Explore, Prompt, Workspace +from core.schemas.explore import (ExploreSchema, ExploreSchemaIn, + ExploreSchemaOut) +from core.schemas.schemas import DetailSchema, SyncStartStopSchemaIn from core.services.explore import ExploreService - +from core.services.prompts import PromptService logger = logging.getLogger(__name__) router = Router() @@ -50,9 +51,9 @@ def get_explores(request, workspace_id): explore.last_sync_succeeded_at = latest_sync_info.created_at explore.last_sync_created_at = latest_sync_info.created_at # adding last successful sync info - # last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) - # if last_successful_sync_info.found == True: - # explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at + last_successful_sync_info = PromptService.is_sync_finished(explore.sync.id) + if last_successful_sync_info.found == True: + explore.last_sync_succeeded_at = last_successful_sync_info.run_end_at return explores except Exception: logger.exception("explores listing error") @@ -91,6 +92,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): account = Account.objects.create(**account_info) data["account"] = account # create source + source = ExploreService.create_source( table_name, data["prompt_id"], data["schema_id"], data["time_window"], data["filters"], data["time_grain"], workspace_id, account) # create destination @@ -112,6 +114,7 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["sync"] = sync data["ready"] = False data["spreadsheet_url"] = spreadsheet_url + del data["time_grain"] explore = Explore.objects.create(**data) # create run asyncio.run(ExploreService.wait_for_run(5)) @@ -132,3 +135,4 @@ def get_explore_by_id(request, workspace_id, explore_id): except Exception: logger.exception("explore listing error") return (500, {"detail": "The explore cannot be fetched."}) + return (500, {"detail": "The explore cannot be fetched."}) diff --git a/core/schemas/explore.py b/core/schemas/explore.py index 1ff57dd..10f417f 100644 --- a/core/schemas/explore.py +++ b/core/schemas/explore.py @@ -1,9 +1,12 @@ from datetime import datetime from typing import Dict, Optional + from ninja import Field, ModelSchema, Schema + from core.models import Explore -from core.schemas.prompt import Filter, TimeWindow, TimeGrain -from core.schemas.schemas import AccountSchema, CamelSchemaConfig, PromptSchema, WorkspaceSchema +from core.schemas.prompt import Filter, TimeGrain, TimeWindow +from core.schemas.schemas import (AccountSchema, CamelSchemaConfig, + PromptSchema, WorkspaceSchema) class ExploreSchemaOut(Schema): @@ -43,10 +46,13 @@ class ExploreSchemaIn(Schema): schema_id: str time_window: TimeWindow filters: list[Filter] - time_grain: Optional[TimeGrain] + time_grain: Optional[TimeGrain] = None class LatestSyncInfo(Schema): found: bool status: Optional[str] = None created_at: Optional[datetime] = None + found: bool + status: Optional[str] = None + created_at: Optional[datetime] = None From f7b231184da59e843ee5007e63a0f8457524725b Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 15:36:55 +0530 Subject: [PATCH 148/159] feat: Added new prompts --- init_db/prompt_def.json | 164 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 6 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 2e14e7a..943ec85 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -86,7 +86,7 @@ }, { "name": "At risk customers", - "description": "Know your customers who haven’t bought anything in the recent past and need a reminder or two", + "description": "Know your customers who haven’t bought anything in the past 90 days and need a reminder or two", "query": "select * from {{schema}}.at_risk_customers where {{filters}}", "filters": [ { @@ -168,12 +168,22 @@ { "name": "Sales by category", "description": "Sales by category", - "query": "inventory_snapshot.sql", + "query": "select * from {{schema}}.sales_by_category where {{filters}}", "filters": [ { - "display_column": "payment_method", - "db_column": "transactions.gateway", + "display_column": "category", + "db_column": "category", "column_type": "string" + }, + { + "display_column": "quantity_sold", + "db_column": "quantity_sold", + "column_type": "integer" + }, + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" } ], "operators": { @@ -187,7 +197,7 @@ { "name": "Sales by payment method", "description": "Get the data of overall sales by different payment methods", - "query": "select Payment_Method, count(*) as Total_Orders, round(sum(total_price)) as Total_Sales from sales_by_payment_method where {{filters}} group by Payment_Method", + "query": "select Payment_Method, count(*) as Total_Orders, round(sum(total_price)) as Total_Sales from {{schema}}.sales_by_payment_method where {{filters}} group by Payment_Method", "filters": [ { "display_column": "payment_method", @@ -203,6 +213,35 @@ "package_id": "P0", "gated": true }, + { + "name": "Orders by date", + "description": "Get the data of overall sales by different payment methods", + "query": "select * from {{schema}}.orders_by_date where {{filters}}", + "filters": [ + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" + }, + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" + }, + { + "display_column": "total_orders", + "db_column": "total_orders", + "column_type": "integer" + } + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, { "name": "Customer Segmentation Builder", "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", @@ -225,7 +264,7 @@ }, { "display_column": "state", - "db_column": "state'", + "db_column": "state", "column_type": "string" }, { @@ -266,6 +305,119 @@ "type": "SRC_SHOPIFY", "package_id": "P0", "gated": true + }, + { + "name": "Order export with products", + "description": "Get the data of overall sales by different payment methods", + "query": "select * from {{schema}}.order_export_with_products where {{filters}}", + "filters": [ + { + "display_column": "name", + "db_column": "name", + "column_type": "string" + }, + { + "display_column": "fulfillment_status", + "db_column": "fulfillment_status", + "column_type": "string" + }, + { + "display_column": "fulfillment_status", + "db_column": "fulfillment_status", + "column_type": "string" + }, + { + "display_column": "email", + "db_column": "email", + "column_type": "string" + }, + { + "display_column": "phone", + "db_column": "phone", + "column_type": "string" + }, + { + "display_column": "shipping_name", + "db_column": "shipping_name", + "column_type": "string" + }, + { + "display_column": "address", + "db_column": "address", + "column_type": "string" + }, + { + "display_column": "shipping_city", + "db_column": "shipping_city", + "column_type": "string" + }, + { + "display_column": "shipping_state", + "db_column": "shipping_state", + "column_type": "string" + }, + { + "display_column": "shipping_country", + "db_column": "shipping_country", + "column_type": "string" + }, + { + "display_column": "shipping_zip", + "db_column": "shipping_zip", + "column_type": "integer" + }, + { + "display_column": "quantity", + "db_column": "quantity", + "column_type": "integer" + }, + { + "display_column": "product_title", + "db_column": "product_title", + "column_type": "string" + }, + { + "display_column": "sku", + "db_column": "sku", + "column_type": "string" + } + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, + { + "name": "New customer sales", + "description": "Get the data of overall sales by different payment methods", + "query": "select * from {{schema}}.new_customer_sales where {{filters}}", + "filters": [ + { + "display_column": "new_customers", + "db_column": "new_customers", + "column_type": "integer" + }, + { + "display_column": "new_customer_sales", + "db_column": "new_customer_sales", + "column_type": "integer" + }, + { + "display_column": "total_customers", + "db_column": "total_customers", + "column_type": "integer" + } + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true } ] } From cb82e5e11f066d4426d75ccdf24a31b3cb6a5454 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 15:38:17 +0530 Subject: [PATCH 149/159] feat: Added new prompts --- core/routes/explore_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/routes/explore_api.py b/core/routes/explore_api.py index 6354cc0..7f5f3ed 100644 --- a/core/routes/explore_api.py +++ b/core/routes/explore_api.py @@ -114,7 +114,6 @@ def create_explore(request, workspace_id, payload: ExploreSchemaIn): data["sync"] = sync data["ready"] = False data["spreadsheet_url"] = spreadsheet_url - del data["time_grain"] explore = Explore.objects.create(**data) # create run asyncio.run(ExploreService.wait_for_run(5)) From a5fea59a9556f6d5a7b52d31d7858b8976203a4c Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 16:21:15 +0530 Subject: [PATCH 150/159] feat: Replaced source names in prompt preview with shopify store names --- core/routes/prompt_api.py | 4 +++- core/routes/workspace_api.py | 4 ++-- init_db/prompt_def.json | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/routes/prompt_api.py b/core/routes/prompt_api.py index e54a278..25a40ae 100644 --- a/core/routes/prompt_api.py +++ b/core/routes/prompt_api.py @@ -52,14 +52,16 @@ def get_prompt_by_id(request, workspace_id, prompt_id): if source_access_info := info.source_access_info.first(): storage_id = source_access_info.storage_credentials.id if storage_id not in schemas: + logger.debug(info.credential.name) schema = { "id": str(storage_id), "name": source_access_info.storage_credentials.connector_config["schema"], "sources": [], } schemas[storage_id] = schema + formatted_output = info.credential.created_at.strftime('%B %d %Y %H:%M') schemas[storage_id]["sources"].append({ - "name": f"{info.credential.name}${info.credential.created_at}", + "name": f"{info.credential.name}@{formatted_output}", "id": str(info.id), }) # Convert schemas dictionary to a list (optional) diff --git a/core/routes/workspace_api.py b/core/routes/workspace_api.py index a93b614..cc90e12 100644 --- a/core/routes/workspace_api.py +++ b/core/routes/workspace_api.py @@ -338,12 +338,12 @@ def create_sync(request, workspace_id, payload: SyncSchemaInWithSourcePayload): stream["destination_sync_mode"] = "append_dedup" # creating source credential source_credential_payload = CredentialSchemaIn( - name="shopify", account=data["account"], connector_type=data["source"]["type"], + name=data["name"], account=data["account"], connector_type=data["source"]["type"], connector_config=data["source"]["config"]) source_credential = create_credential(request, workspace_id, source_credential_payload) # creating source source_payload = SourceSchemaIn( - name="shopify", credential_id=source_credential.id, catalog=catalog) + name=data["name"], credential_id=source_credential.id, catalog=catalog) source = create_source(request, workspace_id, source_payload) workspace = Workspace.objects.get(id=workspace_id) # creating default warehouse diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 943ec85..b18b3cd 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -308,7 +308,7 @@ }, { "name": "Order export with products", - "description": "Get the data of overall sales by different payment methods", + "description": "Get the information of all the orders along with the list of items that have been placed via these orders", "query": "select * from {{schema}}.order_export_with_products where {{filters}}", "filters": [ { @@ -392,7 +392,7 @@ }, { "name": "New customer sales", - "description": "Get the data of overall sales by different payment methods", + "description": "Get the Sales information of new customers in a specific time period", "query": "select * from {{schema}}.new_customer_sales where {{filters}}", "filters": [ { From e3b9f736bf636df883f4ba1806ed62dab50abac5 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 18:24:42 +0530 Subject: [PATCH 151/159] feat: Added field time_window_enabled for prompts --- .../0034_prompt_time_window_enabled.py | 18 ++++++++++ core/models.py | 33 +++++++++++-------- core/schemas/prompt.py | 8 +++-- core/services/prompt_templates/prompts.liquid | 12 ++++--- init_db/prompt_def.json | 3 +- 5 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 core/migrations/0034_prompt_time_window_enabled.py diff --git a/core/migrations/0034_prompt_time_window_enabled.py b/core/migrations/0034_prompt_time_window_enabled.py new file mode 100644 index 0000000..c7d5fc1 --- /dev/null +++ b/core/migrations/0034_prompt_time_window_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.5 on 2024-06-18 11:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0033_prompt_time_grain_enabled'), + ] + + operations = [ + migrations.AddField( + model_name='prompt', + name='time_window_enabled', + field=models.BooleanField(default=True), + ), + ] diff --git a/core/models.py b/core/models.py index d948d33..a60c0bf 100644 --- a/core/models.py +++ b/core/models.py @@ -79,13 +79,14 @@ class Credential(models.Model): def __str__(self): return f"{self.connector}: {self.connector_config} : {self.workspace}: {self.id} : {self.name}" - + class StorageCredentials(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="storage_credentials") connector_config = models.JSONField(blank=False, null=False) + class Source(models.Model): created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) @@ -135,29 +136,34 @@ class Connector(models.Model): class Package(models.Model): - name = models.CharField(primary_key=True,max_length=256, null=False, blank=False) + name = models.CharField(primary_key=True, max_length=256, null=False, blank=False) scopes = ArrayField(models.CharField(max_length=64), blank=True, default=list) - gated = models.BooleanField(null=False, blank = False, default=True) + gated = models.BooleanField(null=False, blank=False, default=True) + class Prompt(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) - name = models.CharField(max_length=256, null=False, blank=False,unique=True) - description = models.CharField(max_length=1000, null=False, blank=False,default="aaaaaa") - type = models.CharField(null=False, blank = False,max_length=256, default="SRC_SHOPIFY") + name = models.CharField(max_length=256, null=False, blank=False, unique=True) + description = models.CharField(max_length=1000, null=False, blank=False, default="aaaaaa") + type = models.CharField(null=False, blank=False, max_length=256, default="SRC_SHOPIFY") # spec = models.JSONField(blank=False, null=True) filters = models.JSONField(default=dict) operators = models.JSONField(default={ 'string': ["=", "!=", "IN", "NOT IN"], 'integer': ["=", ">", "<", ">=", "<=", "!="] }) - query = models.CharField(max_length=1000,null=False, blank=False,default="query") - package_id = models.CharField(null=False, blank = False,max_length=20,default="P0") - gated = models.BooleanField(null=False, blank = False, default=True) + query = models.CharField(max_length=1000, null=False, blank=False, default="query") + package_id = models.CharField(null=False, blank=False, max_length=20, default="P0") + gated = models.BooleanField(null=False, blank=False, default=True) time_grain_enabled = models.BooleanField(null=False, blank=False, default=False) + time_window_enabled = models.BooleanField(null=False, blank=False, default=True) + class SourceAccessInfo(models.Model): - source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info",primary_key=True) - storage_credentials = models.ForeignKey(to=StorageCredentials,on_delete=models.CASCADE,related_name="source_access_info") + source = models.ForeignKey(to=Source, on_delete=models.CASCADE, related_name="source_access_info", primary_key=True) + storage_credentials = models.ForeignKey( + to=StorageCredentials, on_delete=models.CASCADE, related_name="source_access_info") + class Account(models.Model): id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) @@ -172,14 +178,15 @@ class Explore(models.Model): created_at = models.DateTimeField(default=timezone.now) updated_at = models.DateTimeField(auto_now=True) id = models.UUIDField(primary_key=True, editable=False, default=uuid.UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) - name = models.CharField(max_length=256, null=False, blank=False,default="aaaaaa") + name = models.CharField(max_length=256, null=False, blank=False, default="aaaaaa") workspace = models.ForeignKey(to=Workspace, on_delete=models.CASCADE, related_name="explore_workspace") prompt = models.ForeignKey(to=Prompt, on_delete=models.CASCADE, related_name="explore_prompt") sync = models.ForeignKey(to=Sync, on_delete=models.CASCADE, related_name="explore_sync") - ready = models.BooleanField(null=False, blank = False, default=False) + ready = models.BooleanField(null=False, blank=False, default=False) account = models.ForeignKey(to=Account, on_delete=models.CASCADE, related_name="explore_account") spreadsheet_url = models.URLField(null=True, blank=True, default="https://example.com") + class ValmiUserIDJitsuApiToken(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True) api_token = models.CharField(max_length=256, blank=True, null=True) diff --git a/core/schemas/prompt.py b/core/schemas/prompt.py index 7e18c81..dcc1604 100644 --- a/core/schemas/prompt.py +++ b/core/schemas/prompt.py @@ -11,13 +11,15 @@ class TimeWindowRange(Schema): class TimeWindow(Schema): - label: str - range: TimeWindowRange + label: Optional[str] + range: Optional[TimeWindowRange] + class TimeGrain(str, Enum): - day = 'day' + day = 'day' week = 'week' month = 'month' + def members(): return TimeGrain._member_names_ diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 313d618..233377e 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -15,8 +15,12 @@ {%- if filterStr != "" -%} {% assign filterStr = filterStr | append: " and " %} {%- endif -%} -{%- if timeWindow.range.start contains "now()" -%} - {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} +{%- if timeWindow.label != nil and timeWindow.range != nil -%} + {%- if "now()" in timeWindow.range.start -%} + {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} + {%- else -%} + {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' + {%- endif -%} {%- else -%} - {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' -{%- endif -%} \ No newline at end of file + {{ filterStr }} 1=1 +{%- endif -%} diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index b18b3cd..ef55318 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -131,7 +131,8 @@ }, "type": "SRC_SHOPIFY", "package_id": "P0", - "gated": true + "gated": true, + "time_window_enabled":false }, { From 5ad2110ddb946bb45dae3841a4013baf775f98cd Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 18:26:53 +0530 Subject: [PATCH 152/159] feat: Added field time_grain_enabled in prompt schema --- core/schemas/schemas.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 86fd685..6566f2f 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -80,14 +80,16 @@ class Config(CamelSchemaConfig): class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query", "time_grain_enabled"] + model_fields = ["id", "name", "description", "type", "filters", "operators", + "package_id", "gated", "query", "time_grain_enabled", "time_grain_enabled"] class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt - model_fields = ["id", "name", "description", "type", "filters", "operators", "package_id", "gated", "query", "time_grain_enabled"] - time_grain: List[TimeGrain] = None + model_fields = ["id", "name", "description", "type", "filters", "operators", + "package_id", "gated", "query", "time_grain_enabled", "time_grain_enabled"] + time_grain: List[TimeGrain] = None schemas: List[Dict] From 4a93fde44327d365ae428fa354aa61e21a532cc1 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 18:52:06 +0530 Subject: [PATCH 153/159] feat: Added time_window_enabled in schema --- core/schemas/schemas.py | 4 +-- init_db/prompt_def.json | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/core/schemas/schemas.py b/core/schemas/schemas.py index 6566f2f..49970c3 100644 --- a/core/schemas/schemas.py +++ b/core/schemas/schemas.py @@ -81,14 +81,14 @@ class PromptSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt model_fields = ["id", "name", "description", "type", "filters", "operators", - "package_id", "gated", "query", "time_grain_enabled", "time_grain_enabled"] + "package_id", "gated", "query", "time_grain_enabled", "time_window_enabled"] class PromptByIdSchema(ModelSchema): class Config(CamelSchemaConfig): model = Prompt model_fields = ["id", "name", "description", "type", "filters", "operators", - "package_id", "gated", "query", "time_grain_enabled", "time_grain_enabled"] + "package_id", "gated", "query", "time_grain_enabled", "time_window_enabled"] time_grain: List[TimeGrain] = None schemas: List[Dict] diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index ef55318..9b45d7b 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -166,6 +166,60 @@ "gated": true, "time_grain_enabled": true }, + { + "name": "Gross sales", + "description": "Get the list of refunds on any day, filtered by the date", + "query": "select * from {{schema}}.gross_sales where {{filters}}", + "filters": [ + { + "display_column": "orders", + "db_column": "orders", + "column_type": "integer" + }, + { + "display_column": "grosss_sales", + "db_column": "grosss_sales", + "column_type": "integer" + }, + { + "display_column": "discounts", + "db_column": "discounts", + "column_type": "integer" + }, + { + "display_column": "net_sales", + "db_column": "net_sales", + "column_type": "integer" + }, + { + "display_column": "tax", + "db_column": "tax", + "column_type": "integer" + }, + { + "display_column": "total_sales", + "db_column": "total_sales", + "column_type": "integer" + }, + { + "display_column": "shipping_price", + "db_column": "shipping_price", + "column_type": "integer" + }, + { + "display_column": "refund", + "db_column": "refund", + "column_type": "integer" + } + ], + "operators": { + "string": ["=", "!=", "IN", "NOT IN"], + "integer": ["=", ">", "<", ">=", "<=", "!="] + }, + "type": "SRC_SHOPIFY", + "package_id": "P0", + "gated": true + }, { "name": "Sales by category", "description": "Sales by category", From cf86ecdc8686fbab08c250b3a85b74d656cc86c3 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Tue, 18 Jun 2024 19:51:12 +0530 Subject: [PATCH 154/159] feat: Added new prompts --- init_db/prompt_def.json | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 9b45d7b..1ca364c 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -138,21 +138,15 @@ { "name": "Average order value", "description": "Average order value", - "query": "inventory_snapshot.sql", + "query": "select avg(total_price)::NUMERIC(10,2), DATE_TRUNC('{{timegrain}}',o.updated_at) from {{schema}}.orders as o group by DATE_TRUNC('{{timegrain}}',o.updated_at) order by DATE_TRUNC('{{timegrain}}',o.updated_at)", "filters": [ - { - "display_column": "payment_method", - "db_column": "transactions.gateway", - "column_type": "string" - } ], "operators": { - "string": ["=", "!=", "IN", "NOT IN"], - "integer": ["=", ">", "<", ">=", "<=", "!="] }, "type": "SRC_SHOPIFY", "package_id": "P0", - "gated": true + "gated": true, + "time_grain_enabled":true }, { @@ -222,7 +216,7 @@ }, { "name": "Sales by category", - "description": "Sales by category", + "description": "Get a report of the total sales, filtered by the category", "query": "select * from {{schema}}.sales_by_category where {{filters}}", "filters": [ { From 830bf94fd4cc72a87fb316cff1d9de37a37db7b7 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 19 Jun 2024 07:43:36 +0530 Subject: [PATCH 155/159] fix: hacked explore creation --- core/services/prompt_templates/prompts.liquid | 12 ++++++------ core/services/prompts.py | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/services/prompt_templates/prompts.liquid b/core/services/prompt_templates/prompts.liquid index 233377e..0ce6f7b 100644 --- a/core/services/prompt_templates/prompts.liquid +++ b/core/services/prompt_templates/prompts.liquid @@ -15,12 +15,12 @@ {%- if filterStr != "" -%} {% assign filterStr = filterStr | append: " and " %} {%- endif -%} -{%- if timeWindow.label != nil and timeWindow.range != nil -%} - {%- if "now()" in timeWindow.range.start -%} - {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} +{%- if timeWindow.label contains "notimeWindow" -%} + {{ filterStr }} 1=1 +{%- else -%} + {%- if timeWindow.range.start contains "now()" -%} + {{ filterStr }} "updated_at" >= {{ timeWindow.range.start }} and "updated_at" <= {{ timeWindow.range.end }} {%- else -%} - {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' + {{ filterStr }} "updated_at" >= '{{ timeWindow.range.start }}' and "updated_at" <= '{{ timeWindow.range.end }}' {%- endif -%} -{%- else -%} - {{ filterStr }} 1=1 {%- endif -%} diff --git a/core/services/prompts.py b/core/services/prompts.py index 32da54c..26478d1 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -39,6 +39,13 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], t f"{str(template_parent_path)}/prompt_templates" ), ) + # HACK : This neeeds to be done nicely + logger.debug(timeWindowDict) + if 'label' not in timeWindowDict or timeWindowDict['label'] is None: + logger.debug("in none") + timeWindowDict['label'] = 'notimeWindow' + # timeWindow.range = TimeWindowRange(start="empty", end="empty") + logger.debug(timeWindowDict) template = env.get_template(file_name) filters = list(filters) filterList = [filter.dict() for filter in filters] From e411214befeda25c8d2749bf1db431686e34242e Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 19 Jun 2024 11:52:41 +0530 Subject: [PATCH 156/159] feat: added id for refund on daily basis prompt --- init_db/prompt_def.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 1ca364c..f7fe54f 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -138,7 +138,7 @@ { "name": "Average order value", "description": "Average order value", - "query": "select avg(total_price)::NUMERIC(10,2), DATE_TRUNC('{{timegrain}}',o.updated_at) from {{schema}}.orders as o group by DATE_TRUNC('{{timegrain}}',o.updated_at) order by DATE_TRUNC('{{timegrain}}',o.updated_at)", + "query": "select md5(row(DATE_TRUNC('{{timegrain}}', o.updated_at))::text) as id, avg(total_price)::NUMERIC(10,2) as average_total_price, DATE_TRUNC('{{timegrain}}', o.updated_at)::DATE as truncated_date from {{schema}}.orders as o group by DATE_TRUNC('{{timegrain}}', o.updated_at) order by DATE_TRUNC('{{timegrain}}', o.updated_at)::DATE ", "filters": [ ], "operators": { From 8ac3114c77356ac503d7a8f9eef087b124f24f8a Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 19 Jun 2024 12:45:51 +0530 Subject: [PATCH 157/159] fix: resolved explore creation --- core/services/prompts.py | 7 ++++++- init_db/prompt_def.json | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/services/prompts.py b/core/services/prompts.py index 26478d1..e4f429f 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -48,7 +48,12 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], t logger.debug(timeWindowDict) template = env.get_template(file_name) filters = list(filters) - filterList = [filter.dict() for filter in filters] + logger.debug('*' * 80) + logger.debug(type(filters)) + # if isinstance(filters, dict): + # pass + # else: + # filterList = [filter.dict() for filter in filters] rendered_query = template.render(filters=filterList, timeWindow=timeWindowDict) liquid_template = LiquidTemplate(tableInfo.query) context = { diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index f7fe54f..46dc42c 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -146,7 +146,8 @@ "type": "SRC_SHOPIFY", "package_id": "P0", "gated": true, - "time_grain_enabled":true + "time_grain_enabled":true, + "time_window_enabled":false }, { @@ -158,7 +159,8 @@ "type": "SRC_SHOPIFY", "package_id": "P0", "gated": true, - "time_grain_enabled": true + "time_grain_enabled": true, + "time_window_enabled":false }, { "name": "Gross sales", From d7c01ef20ea29681d651286cf517ff50796e19ea Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 19 Jun 2024 12:55:20 +0530 Subject: [PATCH 158/159] feat: Modified descriptions for prompt --- init_db/prompt_def.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/init_db/prompt_def.json b/init_db/prompt_def.json index 46dc42c..0e265e8 100644 --- a/init_db/prompt_def.json +++ b/init_db/prompt_def.json @@ -2,7 +2,7 @@ "definitions": [ { "name": "Inventory snapshot", - "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by products and variants", "query": "select * from {{schema}}.inventory_snapshot where {{filters}}", "filters": [ { @@ -47,7 +47,7 @@ }, { "name": "Abandoned customers", - "description": "List of users who have abandoned their cart", + "description": "Get a list of customers who’ve left their shopping carts unattended after adding items", "query": "select * from {{schema}}.abandon_customers where {{filters}}", "filters": [ { @@ -86,7 +86,7 @@ }, { "name": "At risk customers", - "description": "Know your customers who haven’t bought anything in the past 90 days and need a reminder or two", + "description": "Know your customers who haven’t bought anything in the last 90 days and need a reminder or two", "query": "select * from {{schema}}.at_risk_customers where {{filters}}", "filters": [ { @@ -137,7 +137,7 @@ { "name": "Average order value", - "description": "Average order value", + "description": "Get the average value (in currency terms) of all the orders placed in a time period", "query": "select md5(row(DATE_TRUNC('{{timegrain}}', o.updated_at))::text) as id, avg(total_price)::NUMERIC(10,2) as average_total_price, DATE_TRUNC('{{timegrain}}', o.updated_at)::DATE as truncated_date from {{schema}}.orders as o group by DATE_TRUNC('{{timegrain}}', o.updated_at) order by DATE_TRUNC('{{timegrain}}', o.updated_at)::DATE ", "filters": [ ], @@ -164,7 +164,7 @@ }, { "name": "Gross sales", - "description": "Get the list of refunds on any day, filtered by the date", + "description": "Get the overall picture of all the items sold in a time-frame. This includes parameters like the discounts, taxes, refunds etc", "query": "select * from {{schema}}.gross_sales where {{filters}}", "filters": [ { @@ -266,7 +266,7 @@ }, { "name": "Orders by date", - "description": "Get the data of overall sales by different payment methods", + "description": "Get the total information of orders placed on a given date", "query": "select * from {{schema}}.orders_by_date where {{filters}}", "filters": [ { @@ -295,7 +295,7 @@ }, { "name": "Customer Segmentation Builder", - "description": "Get a snapshot of your active inventory quantity, cost, and total value. Results are grouped by product- and variant-title, archived and draft product variants are ignored.", + "description": "Get to know your customers up and close; filter the customer data based on different demographics and/or sales information; observe this data change over time", "query": "select * from {{schema}}.customer_segmentation where {{filters}}", "filters": [ { @@ -358,7 +358,7 @@ "gated": true }, { - "name": "Order export with products", + "name": "Orders export with products", "description": "Get the information of all the orders along with the list of items that have been placed via these orders", "query": "select * from {{schema}}.order_export_with_products where {{filters}}", "filters": [ From 6630fe2bab9e6a41d2b11a6633068cd48abb77c4 Mon Sep 17 00:00:00 2001 From: supradeep2819 Date: Wed, 19 Jun 2024 13:11:09 +0530 Subject: [PATCH 159/159] fix: handled explore creation --- core/services/prompts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/services/prompts.py b/core/services/prompts.py index e4f429f..cfc9107 100644 --- a/core/services/prompts.py +++ b/core/services/prompts.py @@ -54,6 +54,11 @@ def build(tableInfo: TableInfo, timeWindow: TimeWindow, filters: list[Filter], t # pass # else: # filterList = [filter.dict() for filter in filters] + # HACK: do nicely + try: + filterList = [filter.dict() for filter in filters] + except: + filterList = filters rendered_query = template.render(filters=filterList, timeWindow=timeWindowDict) liquid_template = LiquidTemplate(tableInfo.query) context = {