From d7d738fd3631b3cea246daa956a3ca5297899acc Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Mon, 16 Dec 2024 20:27:06 +0330 Subject: [PATCH 1/7] feat(grafana): add grafana alertmanager --- app/main.py | 3 +- app/media/MyGrafana/alertmanager.yml | 16 +++++ .../grafana_datasources/alertmanager.yml | 21 +++++++ .../grafana_datasources/elasticsearch.yml | 14 +++++ app/media/grafana_datasources/loki.yml | 19 ++++++ app/media/grafana_datasources/mimir.yml | 16 +++++ app/media/grafana_datasources/mysql.yml | 20 ++++++ app/media/grafana_datasources/postgresql.yml | 19 ++++++ app/media/grafana_datasources/prometheus.yml | 23 +++++++ app/media/grafana_datasources/tempo.yml | 25 ++++++++ app/models/__init__.py | 3 +- app/models/alert_managers_models.py | 28 +++++++++ app/routes/grafana_data_sources.py | 17 ++++++ .../grafana_data_sources/alertmanager.py | 61 +++++++++++++++++++ 14 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 app/media/MyGrafana/alertmanager.yml create mode 100644 app/media/grafana_datasources/alertmanager.yml create mode 100644 app/media/grafana_datasources/elasticsearch.yml create mode 100644 app/media/grafana_datasources/loki.yml create mode 100644 app/media/grafana_datasources/mimir.yml create mode 100644 app/media/grafana_datasources/mysql.yml create mode 100644 app/media/grafana_datasources/postgresql.yml create mode 100644 app/media/grafana_datasources/prometheus.yml create mode 100644 app/media/grafana_datasources/tempo.yml create mode 100644 app/models/alert_managers_models.py create mode 100644 app/routes/grafana_data_sources.py create mode 100644 app/template_generators/grafana_data_sources/alertmanager.py diff --git a/app/main.py b/app/main.py index 1980475c..5068cfdf 100644 --- a/app/main.py +++ b/app/main.py @@ -5,4 +5,5 @@ from app.routes.jcasc import * from app.routes.docker import * from app.routes.jenkins import * -from app.routes.gitlab import * \ No newline at end of file +from app.routes.gitlab import * +from app.routes.grafana_data_sources import * \ No newline at end of file diff --git a/app/media/MyGrafana/alertmanager.yml b/app/media/MyGrafana/alertmanager.yml new file mode 100644 index 00000000..5459b49f --- /dev/null +++ b/app/media/MyGrafana/alertmanager.yml @@ -0,0 +1,16 @@ +apiVersion: 1 +datasources: +- name: string + uid: string + type: alertmanager + url: string + access: proxy + orgId: 1 + jsonData: + implementation: cortex + handleGrafanaManagedAlerts: true + editable: true + basicAuth: true + basicAuthUser: string + secureJsonData: + basicAuthPassword: string diff --git a/app/media/grafana_datasources/alertmanager.yml b/app/media/grafana_datasources/alertmanager.yml new file mode 100644 index 00000000..7835d885 --- /dev/null +++ b/app/media/grafana_datasources/alertmanager.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Alertmanager + uid: alertmanager + type: alertmanager + url: http://localhost:9093 + access: proxy + orgId: 1 + jsonData: + # Valid options for implementation include mimir, cortex and prometheus + implementation: [prometheus|cortex|mimir] + # Whether or not Grafana should send alert instances to this Alertmanager + handleGrafanaManagedAlerts: [false|true] + + editable: [true|false] + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/elasticsearch.yml b/app/media/grafana_datasources/elasticsearch.yml new file mode 100644 index 00000000..91a45312 --- /dev/null +++ b/app/media/grafana_datasources/elasticsearch.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: elasticsearch-v7-filebeat + type: elasticsearch + access: proxy + url: http://localhost:9200 + editable: [true|false] + jsonData: + index: '[filebeat-]YYYY.MM.DD' + interval: Daily + timeField: '@timestamp' + logMessageField: message + logLevelField: fields.level diff --git a/app/media/grafana_datasources/loki.yml b/app/media/grafana_datasources/loki.yml new file mode 100644 index 00000000..0dc90212 --- /dev/null +++ b/app/media/grafana_datasources/loki.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + orgId: 1 + access: proxy + editable: [true|false] + url: http://localhost:3100 + jsonData: + timeout: 60 + maxLines: 1000 + + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/mimir.yml b/app/media/grafana_datasources/mimir.yml new file mode 100644 index 00000000..ebbb9c8c --- /dev/null +++ b/app/media/grafana_datasources/mimir.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Mimir + uid: mimir + type: prometheus + access: proxy + orgId: 1 + url: http://mimir-nginx.mimir.svc.cluster.local/prometheus + editable: [true|false] + version: 1 + jsonData: + httpHeaderName1: "X-Scope-OrgID" + alertmanagerUid: "alertmanager" + secureJsonData: + httpHeaderValue1: "pods" diff --git a/app/media/grafana_datasources/mysql.yml b/app/media/grafana_datasources/mysql.yml new file mode 100644 index 00000000..fc5a6fe5 --- /dev/null +++ b/app/media/grafana_datasources/mysql.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: MySQL + type: mysql + url: localhost:3306 + user: grafana + editable: [true|false] + jsonData: + tlsAuth: true + tlsSkipVerify: true + database: grafana + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + secureJsonData: + password: ${GRAFANA_MYSQL_PASSWORD} + tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT} + tlsCACert: ${GRAFANA_TLS_CA_CERT} diff --git a/app/media/grafana_datasources/postgresql.yml b/app/media/grafana_datasources/postgresql.yml new file mode 100644 index 00000000..bdd8b891 --- /dev/null +++ b/app/media/grafana_datasources/postgresql.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Postgres + type: postgres + url: localhost:5432 + user: grafana # Database user’s login/username + editable: [true|false] + secureJsonData: + password: 'Password!' + jsonData: + database: grafana + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false diff --git a/app/media/grafana_datasources/prometheus.yml b/app/media/grafana_datasources/prometheus.yml new file mode 100644 index 00000000..657a1c98 --- /dev/null +++ b/app/media/grafana_datasources/prometheus.yml @@ -0,0 +1,23 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + # Access mode - proxy (server in the UI) or direct (browser in the UI). + url: http://localhost:9090 + editable: [true|false] + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.44.0 + cacheLevel: 'High' + disableRecordingRules: false + incrementalQueryOverlapWindow: 10m + exemplarTraceIdDestinations: + # Field with internal link pointing to data source in Grafana. + # datasourceUid value can be anything, but it should be unique across all defined data source uids. + - datasourceUid: my_jaeger_uid + name: traceID diff --git a/app/media/grafana_datasources/tempo.yml b/app/media/grafana_datasources/tempo.yml new file mode 100644 index 00000000..cb8794ce --- /dev/null +++ b/app/media/grafana_datasources/tempo.yml @@ -0,0 +1,25 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo-query-frontend.tempo.svc.cluster.local:3100 + basicAuth: false + version: 1 + editable: true + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: # If you are going to link your tracing data with logs, configure <> + datasourceUid: 'loki' + spanStartTimeShift: '-2m' + spanEndTimeShift: '2m' + filterByTraceID: true + filterBySpanID: true + serviceMap: # If you are going to add serviceGraph feature to tempo, configure <> + datasourceUid: 'Mimir-OtelMetrics-Tenant' + nodeGraph: # If you are going to add nodeGraph feature to tempo, enable <> + enabled: true diff --git a/app/models/__init__.py b/app/models/__init__.py index a529a87f..a819a217 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -6,4 +6,5 @@ from .compose_models import * from .docker_installation_models import * from .jenkins import * -from .gitlab_models import * \ No newline at end of file +from .gitlab_models import * +from .alert_managers_models import * \ No newline at end of file diff --git a/app/models/alert_managers_models.py b/app/models/alert_managers_models.py new file mode 100644 index 00000000..29539ccc --- /dev/null +++ b/app/models/alert_managers_models.py @@ -0,0 +1,28 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + +class BasicAuth(BaseModel): + basicAuthUser:str + basicAuthPassword:str + + +class AlertManagerInput(BaseModel): + name:str + url:str + uid:str + implementation:str + + handleGrafanaManagedAlerts:bool = True + editable: bool = True + basic_auth:Optional[BasicAuth] + + @validator("implementation") + def validator_implementation(cls,value): + valid = ['prometheus','cortex','mimir'] + if value not in valid: + raise ValueError(f"implementation must be in {valid}") + return value + + + + diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py new file mode 100644 index 00000000..9a7c22a7 --- /dev/null +++ b/app/routes/grafana_data_sources.py @@ -0,0 +1,17 @@ +from app.app_instance import app +from app.models import (AlertManagerInput,Output) +from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template +import shutil +import os + +@app.post("/api/grafana/alertmanager") +async def alertmanager_template(request:AlertManagerInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + alert_manager_template(request) + + return Output(output='output') + diff --git a/app/template_generators/grafana_data_sources/alertmanager.py b/app/template_generators/grafana_data_sources/alertmanager.py new file mode 100644 index 00000000..694783e7 --- /dev/null +++ b/app/template_generators/grafana_data_sources/alertmanager.py @@ -0,0 +1,61 @@ +import yaml +import os + +def alert_manager_template(input): + if input.basic_auth is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + "basicAuth": True, + "basicAuthUser": input.basic_auth.basicAuthUser, + "secureJsonData": { + "basicAuthPassword": input.basic_auth.basicAuthPassword + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) From 08876b1e1dd532b17a0f27f8ee4b3897e3bc646c Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Mon, 16 Dec 2024 21:20:07 +0330 Subject: [PATCH 2/7] feat(grafana): add ES and loki --- app/media/MyGrafana/alertmanager.yml | 16 ----- app/media/MyGrafana/loki.yml | 12 ++++ app/models/__init__.py | 4 +- app/models/grafana/__init__.py | 0 .../{ => grafana}/alert_managers_models.py | 0 app/models/grafana/elastcsearch_models.py | 21 ++++++ app/models/grafana/loki_models.py | 20 ++++++ app/routes/grafana_data_sources.py | 25 +++++++- .../grafana_data_sources/elasticsearch.py | 36 +++++++++++ .../grafana_data_sources/loki.py | 64 +++++++++++++++++++ 10 files changed, 180 insertions(+), 18 deletions(-) delete mode 100644 app/media/MyGrafana/alertmanager.yml create mode 100644 app/media/MyGrafana/loki.yml create mode 100644 app/models/grafana/__init__.py rename app/models/{ => grafana}/alert_managers_models.py (100%) create mode 100644 app/models/grafana/elastcsearch_models.py create mode 100644 app/models/grafana/loki_models.py create mode 100644 app/template_generators/grafana_data_sources/elasticsearch.py create mode 100644 app/template_generators/grafana_data_sources/loki.py diff --git a/app/media/MyGrafana/alertmanager.yml b/app/media/MyGrafana/alertmanager.yml deleted file mode 100644 index 5459b49f..00000000 --- a/app/media/MyGrafana/alertmanager.yml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: 1 -datasources: -- name: string - uid: string - type: alertmanager - url: string - access: proxy - orgId: 1 - jsonData: - implementation: cortex - handleGrafanaManagedAlerts: true - editable: true - basicAuth: true - basicAuthUser: string - secureJsonData: - basicAuthPassword: string diff --git a/app/media/MyGrafana/loki.yml b/app/media/MyGrafana/loki.yml new file mode 100644 index 00000000..d5323f12 --- /dev/null +++ b/app/media/MyGrafana/loki.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +datasources: +- name: string + uid: loki + type: loki + orgId: 1 + url: string + access: proxy + jsonData: + timeout: 60 + maxLines: 1000 + editable: true diff --git a/app/models/__init__.py b/app/models/__init__.py index a819a217..491b4fa7 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -7,4 +7,6 @@ from .docker_installation_models import * from .jenkins import * from .gitlab_models import * -from .alert_managers_models import * \ No newline at end of file +from app.models.grafana.alert_managers_models import * +from app.models.grafana.elastcsearch_models import * +from app.models.grafana.loki_models import * \ No newline at end of file diff --git a/app/models/grafana/__init__.py b/app/models/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/alert_managers_models.py b/app/models/grafana/alert_managers_models.py similarity index 100% rename from app/models/alert_managers_models.py rename to app/models/grafana/alert_managers_models.py diff --git a/app/models/grafana/elastcsearch_models.py b/app/models/grafana/elastcsearch_models.py new file mode 100644 index 00000000..c0f9d589 --- /dev/null +++ b/app/models/grafana/elastcsearch_models.py @@ -0,0 +1,21 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class ElasticSearchInput(BaseModel): + name:str + url:str + editable: bool = True + index:str = "[filebeat-]YYYY.MM.DD" + interval:str = "Daily" + timeField:str = "@timestamp" + logMessageField:str = "message" + logLevelField:str = "fields.level" + + + + + + + \ No newline at end of file diff --git a/app/models/grafana/loki_models.py b/app/models/grafana/loki_models.py new file mode 100644 index 00000000..e56fbb91 --- /dev/null +++ b/app/models/grafana/loki_models.py @@ -0,0 +1,20 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + +class BasicAuth(BaseModel): + basicAuthUser:str + basicAuthPassword:str + +class LokiInput(BaseModel): + name:str + uid:str ="loki" + url:str + editable: bool = True + timeout:int = 60 + maxLines:int = 1000 + basic_auth:Optional[BasicAuth] + + + + + \ No newline at end of file diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py index 9a7c22a7..b3d430ae 100644 --- a/app/routes/grafana_data_sources.py +++ b/app/routes/grafana_data_sources.py @@ -1,6 +1,8 @@ from app.app_instance import app -from app.models import (AlertManagerInput,Output) +from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput) from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template +from app.template_generators.grafana_data_sources.elasticsearch import elasticsearch_template +from app.template_generators.grafana_data_sources.loki import loki_template import shutil import os @@ -15,3 +17,24 @@ async def alertmanager_template(request:AlertManagerInput) -> Output: return Output(output='output') +@app.post("/api/grafana/elasticsearch") +async def elastic_template(request:ElasticSearchInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + elasticsearch_template(request) + + return Output(output='output') + +@app.post("/api/grafana/loki") +async def elastic_template(request:LokiInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + loki_template(request) + + return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/elasticsearch.py b/app/template_generators/grafana_data_sources/elasticsearch.py new file mode 100644 index 00000000..92518455 --- /dev/null +++ b/app/template_generators/grafana_data_sources/elasticsearch.py @@ -0,0 +1,36 @@ +import yaml +import os + + +def elasticsearch_template(input): + + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "elasticsearch", + "url": input.url, + "access": "proxy", + + "jsonData": { + "index": input.index, + "interval": input.interval, + "timeField": input.timeField, + "logMessageField": input.logMessageField, + "logLevelField": input.logLevelField, + + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'elasticsearch.yml') + + file=open("app/media/MyGrafana/elasticsearch.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/loki.py b/app/template_generators/grafana_data_sources/loki.py new file mode 100644 index 00000000..0e5ab794 --- /dev/null +++ b/app/template_generators/grafana_data_sources/loki.py @@ -0,0 +1,64 @@ +import yaml +import os + +def loki_template(input): + if input.basic_auth is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "loki", + "orgId": 1, + "url": input.url, + "access": "proxy", + + "jsonData": { + "timeout": input.timeout, + "maxLines": input.maxLines + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'loki.yml') + + file=open("app/media/MyGrafana/loki.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "loki", + "url": input.url, + "access": "proxy", + "orgId": 1, + + "jsonData": { + "timeout": input.timeout, + "maxLines": input.maxLines + }, + "editable": input.editable, + "basicAuth": True, + "basicAuthUser": input.basic_auth.basicAuthUser, + "secureJsonData": { + "basicAuthPassword": input.basic_auth.basicAuthPassword + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'loki.yml') + + file=open("app/media/MyGrafana/loki.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + From 38d36f63a49eaafee2a95391af64bb3e129d729e Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Mon, 16 Dec 2024 21:38:14 +0330 Subject: [PATCH 3/7] fix(grafana): add default values --- app/models/grafana/alert_managers_models.py | 8 ++++---- app/models/grafana/elastcsearch_models.py | 4 ++-- app/models/grafana/loki_models.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/models/grafana/alert_managers_models.py b/app/models/grafana/alert_managers_models.py index 29539ccc..595e99f6 100644 --- a/app/models/grafana/alert_managers_models.py +++ b/app/models/grafana/alert_managers_models.py @@ -7,10 +7,10 @@ class BasicAuth(BaseModel): class AlertManagerInput(BaseModel): - name:str - url:str - uid:str - implementation:str + name:str = "Alertmanager" + url:str = "http://localhost:9093" + uid:str = "alertmanager" + implementation:str handleGrafanaManagedAlerts:bool = True editable: bool = True diff --git a/app/models/grafana/elastcsearch_models.py b/app/models/grafana/elastcsearch_models.py index c0f9d589..d8273b8c 100644 --- a/app/models/grafana/elastcsearch_models.py +++ b/app/models/grafana/elastcsearch_models.py @@ -4,8 +4,8 @@ class ElasticSearchInput(BaseModel): - name:str - url:str + name:str = "elasticsearch-v7-filebeat" + url:str = "http://localhost:9200" editable: bool = True index:str = "[filebeat-]YYYY.MM.DD" interval:str = "Daily" diff --git a/app/models/grafana/loki_models.py b/app/models/grafana/loki_models.py index e56fbb91..43318c86 100644 --- a/app/models/grafana/loki_models.py +++ b/app/models/grafana/loki_models.py @@ -6,9 +6,9 @@ class BasicAuth(BaseModel): basicAuthPassword:str class LokiInput(BaseModel): - name:str - uid:str ="loki" - url:str + name:str = "Loki" + uid:str = "loki" + url:str = "http://localhost:3100" editable: bool = True timeout:int = 60 maxLines:int = 1000 From aa387a409874150c72515a352ca8642149998804 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 17 Dec 2024 14:31:16 +0330 Subject: [PATCH 4/7] feat(grafana): add mysql and mimir --- app/media/MyGrafana/loki.yml | 12 --- app/media/MyGrafana/mysql.yml | 19 +++++ app/models/__init__.py | 7 +- app/models/grafana/mimir_models.py | 16 ++++ app/models/grafana/mysql_models.py | 25 +++++++ app/models/grafana/postgresql_models.py | 0 app/models/grafana/prometheus_models.py | 0 app/models/grafana/tempo_models.py | 0 app/routes/grafana_data_sources.py | 33 +++++++- .../grafana_data_sources/mimir.py | 36 +++++++++ .../grafana_data_sources/mysql.py | 75 +++++++++++++++++++ .../grafana_data_sources/postgresql.py | 0 .../grafana_data_sources/prometheus.py | 0 .../grafana_data_sources/tempo.py | 0 14 files changed, 206 insertions(+), 17 deletions(-) delete mode 100644 app/media/MyGrafana/loki.yml create mode 100644 app/media/MyGrafana/mysql.yml create mode 100644 app/models/grafana/mimir_models.py create mode 100644 app/models/grafana/mysql_models.py create mode 100644 app/models/grafana/postgresql_models.py create mode 100644 app/models/grafana/prometheus_models.py create mode 100644 app/models/grafana/tempo_models.py create mode 100644 app/template_generators/grafana_data_sources/mimir.py create mode 100644 app/template_generators/grafana_data_sources/mysql.py create mode 100644 app/template_generators/grafana_data_sources/postgresql.py create mode 100644 app/template_generators/grafana_data_sources/prometheus.py create mode 100644 app/template_generators/grafana_data_sources/tempo.py diff --git a/app/media/MyGrafana/loki.yml b/app/media/MyGrafana/loki.yml deleted file mode 100644 index d5323f12..00000000 --- a/app/media/MyGrafana/loki.yml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: 1 -datasources: -- name: string - uid: loki - type: loki - orgId: 1 - url: string - access: proxy - jsonData: - timeout: 60 - maxLines: 1000 - editable: true diff --git a/app/media/MyGrafana/mysql.yml b/app/media/MyGrafana/mysql.yml new file mode 100644 index 00000000..28ff7545 --- /dev/null +++ b/app/media/MyGrafana/mysql.yml @@ -0,0 +1,19 @@ +apiVersion: 1 +datasources: +- name: MySQL + type: mysql + url: ' localhost:3306' + user: grafana + editable: true + jsonData: + tlsAuth: true + tlsSkipVerify: true + database: grafana + maxOpenConns: 100 + maxIdleConns: 100 + maxIdleConnsAuto: true + connMaxLifetime: 14400 + secureJsonData: + password: ${GRAFANA_MYSQL_PASSWORD} + tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT} + tlsCACert: ${GRAFANA_TLS_CA_CERT} diff --git a/app/models/__init__.py b/app/models/__init__.py index 491b4fa7..d3b34264 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -9,4 +9,9 @@ from .gitlab_models import * from app.models.grafana.alert_managers_models import * from app.models.grafana.elastcsearch_models import * -from app.models.grafana.loki_models import * \ No newline at end of file +from app.models.grafana.loki_models import * +from app.models.grafana.mimir_models import * +from app.models.grafana.mysql_models import * +from app.models.grafana.postgresql_models import * +from app.models.grafana.prometheus_models import * +from app.models.grafana.tempo_models import * \ No newline at end of file diff --git a/app/models/grafana/mimir_models.py b/app/models/grafana/mimir_models.py new file mode 100644 index 00000000..9acef1e5 --- /dev/null +++ b/app/models/grafana/mimir_models.py @@ -0,0 +1,16 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + + +class MimirInput(BaseModel): + name:str = "Mimir" + uid:str = "mimir" + url:str = "http://mimir-nginx.mimir.svc.cluster.local/prometheus" + editable: bool = True + httpHeaderName1:str = "X-Scope-OrgID" + alertmanagerUid:str = "alertmanager" + httpHeaderValue1:str = "pods" + + \ No newline at end of file diff --git a/app/models/grafana/mysql_models.py b/app/models/grafana/mysql_models.py new file mode 100644 index 00000000..a2587861 --- /dev/null +++ b/app/models/grafana/mysql_models.py @@ -0,0 +1,25 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class TLS(BaseModel): + tlsClientCert:str = "${GRAFANA_TLS_CLIENT_CERT}" + tlsCACert:str = "${GRAFANA_TLS_CA_CERT}" + tlsAuth:bool = True + tlsSkipVerify:bool = True + +class MysqlInput(BaseModel): + name:str = "MySQL" + url:str = " localhost:3306" + user:str = "grafana" + editable: bool = True + database:str = "grafana" + maxOpenConns:int = 100 + maxIdleConns:int = 100 + maxIdleConnsAuto:bool = True + connMaxLifetime:int = 14400 + password:str = "${GRAFANA_MYSQL_PASSWORD}" + tls :Optional[TLS] + + \ No newline at end of file diff --git a/app/models/grafana/postgresql_models.py b/app/models/grafana/postgresql_models.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/grafana/prometheus_models.py b/app/models/grafana/prometheus_models.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/grafana/tempo_models.py b/app/models/grafana/tempo_models.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py index b3d430ae..c9c40e0f 100644 --- a/app/routes/grafana_data_sources.py +++ b/app/routes/grafana_data_sources.py @@ -1,13 +1,15 @@ from app.app_instance import app -from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput) +from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput,MimirInput,MysqlInput) from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template from app.template_generators.grafana_data_sources.elasticsearch import elasticsearch_template from app.template_generators.grafana_data_sources.loki import loki_template +from app.template_generators.grafana_data_sources.mimir import mimir_template +from app.template_generators.grafana_data_sources.mysql import mysql_template import shutil import os @app.post("/api/grafana/alertmanager") -async def alertmanager_template(request:AlertManagerInput) -> Output: +async def alertmanager_template_route(request:AlertManagerInput) -> Output: dir = 'app/media/MyGrafana' if os.path.exists(dir): @@ -18,7 +20,7 @@ async def alertmanager_template(request:AlertManagerInput) -> Output: return Output(output='output') @app.post("/api/grafana/elasticsearch") -async def elastic_template(request:ElasticSearchInput) -> Output: +async def elastic_template_route(request:ElasticSearchInput) -> Output: dir = 'app/media/MyGrafana' if os.path.exists(dir): @@ -29,7 +31,7 @@ async def elastic_template(request:ElasticSearchInput) -> Output: return Output(output='output') @app.post("/api/grafana/loki") -async def elastic_template(request:LokiInput) -> Output: +async def loki_template_route(request:LokiInput) -> Output: dir = 'app/media/MyGrafana' if os.path.exists(dir): @@ -37,4 +39,27 @@ async def elastic_template(request:LokiInput) -> Output: loki_template(request) + return Output(output='output') + + +@app.post("/api/grafana/mimir") +async def mimir_template_route(request:MimirInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + mimir_template(request) + + return Output(output='output') + +@app.post("/api/grafana/mysql") +async def mysql_template_route(request:MysqlInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + mysql_template(request) + return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/mimir.py b/app/template_generators/grafana_data_sources/mimir.py new file mode 100644 index 00000000..64122a10 --- /dev/null +++ b/app/template_generators/grafana_data_sources/mimir.py @@ -0,0 +1,36 @@ +import yaml +import os + +def mimir_template(input): + + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "prometheus", + "access": "proxy", + "orgId": 1, + "url": input.url, + "editable": input.editable, + "version": 1, + "jsonData": { + "httpHeaderName1": input.httpHeaderName1, + "alertmanagerUid": input.alertmanagerUid + }, + "secureJsonData": { + "httpHeaderValue1": input.httpHeaderValue1 + } + } + ] + } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mimir.yml') + + file=open("app/media/MyGrafana/mimir.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/mysql.py b/app/template_generators/grafana_data_sources/mysql.py new file mode 100644 index 00000000..4a736ffc --- /dev/null +++ b/app/template_generators/grafana_data_sources/mysql.py @@ -0,0 +1,75 @@ +import yaml +import os + +def mysql_template(input): + + if input.tls is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "mysql", + "url": input.url, + "user": input.user, + "editable": input.editable, + "jsonData": { + + "database": input.database, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns":input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime + }, + "secureJsonData": { + "password": input.password, + + } + } + ] + } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mysql.yml') + + file=open("app/media/MyGrafana/mysql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "mysql", + "url": input.url, + "user": input.user, + "editable": input.editable, + "jsonData": { + "tlsAuth": input.tls.tlsAuth, + "tlsSkipVerify": input.tls.tlsSkipVerify, + "database": input.database, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns":input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime + }, + "secureJsonData": { + + "password": input.password, + "tlsClientCert": input.tls.tlsClientCert, + "tlsCACert": input.tls.tlsCACert + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mysql.yml') + + file=open("app/media/MyGrafana/mysql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/postgresql.py b/app/template_generators/grafana_data_sources/postgresql.py new file mode 100644 index 00000000..e69de29b diff --git a/app/template_generators/grafana_data_sources/prometheus.py b/app/template_generators/grafana_data_sources/prometheus.py new file mode 100644 index 00000000..e69de29b diff --git a/app/template_generators/grafana_data_sources/tempo.py b/app/template_generators/grafana_data_sources/tempo.py new file mode 100644 index 00000000..e69de29b From fe5fbc6e1835da1cb8f502097d831e0260bd870e Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 17 Dec 2024 16:44:09 +0330 Subject: [PATCH 5/7] feat(grafana): compelete grafana data sources --- app/media/MyGrafana/mysql.yml | 19 --------- app/media/MyGrafana/tempo.yaml | 16 ++++++++ app/models/grafana/postgresql_models.py | 22 ++++++++++ app/models/grafana/prometheus_models.py | 19 +++++++++ app/models/grafana/tempo_models.py | 39 ++++++++++++++++++ app/routes/grafana_data_sources.py | 40 +++++++++++++++++- .../grafana_data_sources/postgresql.py | 38 +++++++++++++++++ .../grafana_data_sources/prometheus.py | 41 +++++++++++++++++++ .../grafana_data_sources/tempo.py | 23 +++++++++++ 9 files changed, 237 insertions(+), 20 deletions(-) delete mode 100644 app/media/MyGrafana/mysql.yml create mode 100644 app/media/MyGrafana/tempo.yaml diff --git a/app/media/MyGrafana/mysql.yml b/app/media/MyGrafana/mysql.yml deleted file mode 100644 index 28ff7545..00000000 --- a/app/media/MyGrafana/mysql.yml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: 1 -datasources: -- name: MySQL - type: mysql - url: ' localhost:3306' - user: grafana - editable: true - jsonData: - tlsAuth: true - tlsSkipVerify: true - database: grafana - maxOpenConns: 100 - maxIdleConns: 100 - maxIdleConnsAuto: true - connMaxLifetime: 14400 - secureJsonData: - password: ${GRAFANA_MYSQL_PASSWORD} - tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT} - tlsCACert: ${GRAFANA_TLS_CA_CERT} diff --git a/app/media/MyGrafana/tempo.yaml b/app/media/MyGrafana/tempo.yaml new file mode 100644 index 00000000..a2b4fcd3 --- /dev/null +++ b/app/media/MyGrafana/tempo.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 +datasources: +- name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo-query-frontend.tempo.svc.cluster.local:3100 + basicAuth: false + version: 1 + editable: true + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: Mimir-OtelMetrics-Tenant diff --git a/app/models/grafana/postgresql_models.py b/app/models/grafana/postgresql_models.py index e69de29b..0850e700 100644 --- a/app/models/grafana/postgresql_models.py +++ b/app/models/grafana/postgresql_models.py @@ -0,0 +1,22 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + + +class PostgresInput(BaseModel): + name:str = "Postgres" + url:str = "localhost:5432" + user:str = "grafana" + editable: bool = True + database:str = "grafana" + sslmode:str = "disable" + password:str = "Password!" + maxOpenConns:int = 100 + maxIdleConns:int = 100 + maxIdleConnsAuto:bool = True + connMaxLifetime:int = 14400 + postgresVersion:int = 903 + timescaledb:bool = False + + \ No newline at end of file diff --git a/app/models/grafana/prometheus_models.py b/app/models/grafana/prometheus_models.py index e69de29b..3a04eedb 100644 --- a/app/models/grafana/prometheus_models.py +++ b/app/models/grafana/prometheus_models.py @@ -0,0 +1,19 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + +class TraceID(BaseModel): + datasourceUid:str = "my_jaeger_uid" + name:str = "traceID" +class PrometheusInput(BaseModel): + name:str = "Prometheus" + url:str = "http://localhost:9090" + editable: bool = True + httpMethod:str = "POST" + manageAlerts:bool = True + prometheusType:str = "Prometheus" + prometheusVersion:str = "2.44.0" + cacheLevel:str = 'High' + disableRecordingRules:bool = False + incrementalQueryOverlapWindow:str = "10m" + trace_id: TraceID \ No newline at end of file diff --git a/app/models/grafana/tempo_models.py b/app/models/grafana/tempo_models.py index e69de29b..8267d2f3 100644 --- a/app/models/grafana/tempo_models.py +++ b/app/models/grafana/tempo_models.py @@ -0,0 +1,39 @@ +from typing import Optional, List +from pydantic import BaseModel,PrivateAttr,Field + +class TracesToLogsV2(BaseModel): + datasourceUid: str = 'loki' + spanStartTimeShift: str = '-2m' + spanEndTimeShift: str = '2m' + filterByTraceID: bool = True + filterBySpanID: bool = True + +class ServiceMap(BaseModel): + datasourceUid: str = 'Mimir-OtelMetrics-Tenant' + +class NodeGraph(BaseModel): + enabled: bool = True + +class JsonData(BaseModel): + httpMethod: str = 'GET' + tracesToLogsV2: Optional[TracesToLogsV2] = TracesToLogsV2() + serviceMap: Optional[ServiceMap] = ServiceMap() + nodeGraph: Optional[NodeGraph] = NodeGraph() + +class Datasource(BaseModel): + name: str = 'Tempo' + type: str = Field(default='tempo') + access: str = Field(default="proxy") + orgId: int = Field(default=1) + url: str = 'http://tempo-query-frontend.tempo.svc.cluster.local:3100' + basicAuth: bool = False + version: int = Field(default=1) + editable: bool = True + apiVersion: int = Field(default=1) + uid: str = Field(default="tempo") + jsonData: JsonData = JsonData() + +class TempoInput(BaseModel): + apiVersion: int = Field(default=1) + datasources: List[Datasource] = [Datasource()] + diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py index c9c40e0f..57eb7d16 100644 --- a/app/routes/grafana_data_sources.py +++ b/app/routes/grafana_data_sources.py @@ -1,10 +1,14 @@ from app.app_instance import app -from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput,MimirInput,MysqlInput) +from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput,MimirInput,MysqlInput,PostgresInput, + PrometheusInput,TempoInput) from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template from app.template_generators.grafana_data_sources.elasticsearch import elasticsearch_template from app.template_generators.grafana_data_sources.loki import loki_template from app.template_generators.grafana_data_sources.mimir import mimir_template from app.template_generators.grafana_data_sources.mysql import mysql_template +from app.template_generators.grafana_data_sources.postgresql import postgres_template +from app.template_generators.grafana_data_sources.prometheus import pormetheus_template +from app.template_generators.grafana_data_sources.tempo import tempo_template import shutil import os @@ -62,4 +66,38 @@ async def mysql_template_route(request:MysqlInput) -> Output: mysql_template(request) + return Output(output='output') + +@app.post("/api/grafana/postgres") +async def postgres_template_route(request:PostgresInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + postgres_template(request) + + return Output(output='output') + +@app.post("/api/grafana/prometheus") +async def prometheus_template_route(request:PrometheusInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + pormetheus_template(request) + + return Output(output='output') + + +@app.post("/api/grafana/tempo") +async def tempo_template_route(request:TempoInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + tempo_template(request) + return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/postgresql.py b/app/template_generators/grafana_data_sources/postgresql.py index e69de29b..2e2de924 100644 --- a/app/template_generators/grafana_data_sources/postgresql.py +++ b/app/template_generators/grafana_data_sources/postgresql.py @@ -0,0 +1,38 @@ +import yaml +import os + +def postgres_template(input): + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "postgres", + "url": input.url, + "user": input.user, + "editable": input.editable, + "secureJsonData": { + "password": input.password + }, + "jsonData": { + "database": input.database, + "sslmode": input.sslmode, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns": input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime, + "postgresVersion": input.postgresVersion, + "timescaledb": input.timescaledb + } + } + ] + } + + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'postgresql.yml') + + file=open("app/media/MyGrafana/postgresql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/prometheus.py b/app/template_generators/grafana_data_sources/prometheus.py index e69de29b..ef8afa18 100644 --- a/app/template_generators/grafana_data_sources/prometheus.py +++ b/app/template_generators/grafana_data_sources/prometheus.py @@ -0,0 +1,41 @@ +import yaml +import os + +def pormetheus_template(input): + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": "prometheus", + "type": "prometheus", + "access": "proxy", + "url": input.url, + "editable": input.editable, + "jsonData": { + "httpMethod": input.httpMethod, + "manageAlerts": input.manageAlerts, + "prometheusType": input.prometheusType, + "prometheusVersion": input.prometheusVersion, + "cacheLevel": input.cacheLevel, + "disableRecordingRules": input.disableRecordingRules, + "incrementalQueryOverlapWindow": input.incrementalQueryOverlapWindow, + "exemplarTraceIdDestinations": [ + { + "datasourceUid": input.trace_id.datasourceUid, + "name": input.trace_id.name + } + ] + } + } + ] + } + + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'prometheus.yml') + + file=open("app/media/MyGrafana/prometheus.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/tempo.py b/app/template_generators/grafana_data_sources/tempo.py index e69de29b..e962f146 100644 --- a/app/template_generators/grafana_data_sources/tempo.py +++ b/app/template_generators/grafana_data_sources/tempo.py @@ -0,0 +1,23 @@ +import yaml +import os + +def remove_none_values(d): + if isinstance(d, dict): + return {k: remove_none_values(v) for k, v in d.items() if v is not None} + elif isinstance(d, list): + return [remove_none_values(i) for i in d if i is not None] + return d + +def tempo_template(input): + dir = 'app/media/MyGrafana' + compose_total = input.dict(exclude_none=True) + + + os.makedirs(dir) + os.path.join(dir, 'tempo.yaml') + + file=open("app/media/MyGrafana/tempo.yaml","w") + yaml.dump(compose_total,file,default_flow_style=False, sort_keys=False) + file.close() + + \ No newline at end of file From 446446f7c78429cb3b0a1178e45db442acb17851 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 17 Dec 2024 22:09:23 +0330 Subject: [PATCH 6/7] fix(mimir): fix mimir multitenency --- app/media/MyGrafana/mimir.yml | 13 +++ app/media/MyGrafana/tempo.yaml | 16 ---- app/models/grafana/mimir_models.py | 8 +- app/models/grafana/mysql_models.py | 2 +- app/models/grafana/postgresql_models.py | 2 +- app/models/grafana/prometheus_models.py | 10 +-- app/routes/ansible.py | 2 +- app/routes/helm.py | 2 +- app/routes/jcasc.py | 2 +- app/routes/terraform.py | 2 +- app/services.py | 31 +------ .../grafana_data_sources/mimir.py | 86 +++++++++++++------ .../grafana_data_sources/prometheus.py | 18 ++-- app/utils.py | 37 -------- requirements.txt | 3 +- 15 files changed, 99 insertions(+), 135 deletions(-) create mode 100644 app/media/MyGrafana/mimir.yml delete mode 100644 app/media/MyGrafana/tempo.yaml diff --git a/app/media/MyGrafana/mimir.yml b/app/media/MyGrafana/mimir.yml new file mode 100644 index 00000000..01502a86 --- /dev/null +++ b/app/media/MyGrafana/mimir.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +datasources: +- name: Mimir + uid: mimir + type: prometheus + access: proxy + orgId: 1 + url: http://mimir-nginx.mimir.svc.cluster.local/prometheus + editable: true + version: 1 + jsonData: + httpHeaderName1: X-Scope-OrgID + alertmanagerUid: alertmanager diff --git a/app/media/MyGrafana/tempo.yaml b/app/media/MyGrafana/tempo.yaml deleted file mode 100644 index a2b4fcd3..00000000 --- a/app/media/MyGrafana/tempo.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: 1 -datasources: -- name: Tempo - type: tempo - access: proxy - orgId: 1 - url: http://tempo-query-frontend.tempo.svc.cluster.local:3100 - basicAuth: false - version: 1 - editable: true - apiVersion: 1 - uid: tempo - jsonData: - httpMethod: GET - serviceMap: - datasourceUid: Mimir-OtelMetrics-Tenant diff --git a/app/models/grafana/mimir_models.py b/app/models/grafana/mimir_models.py index 9acef1e5..13d0f616 100644 --- a/app/models/grafana/mimir_models.py +++ b/app/models/grafana/mimir_models.py @@ -3,7 +3,9 @@ - +class MultiTenancy(BaseModel): + tenant_name:str = "pods" + class MimirInput(BaseModel): name:str = "Mimir" uid:str = "mimir" @@ -11,6 +13,6 @@ class MimirInput(BaseModel): editable: bool = True httpHeaderName1:str = "X-Scope-OrgID" alertmanagerUid:str = "alertmanager" - httpHeaderValue1:str = "pods" + multi_tenancy:Optional[MultiTenancy] - \ No newline at end of file + diff --git a/app/models/grafana/mysql_models.py b/app/models/grafana/mysql_models.py index a2587861..c3ac157b 100644 --- a/app/models/grafana/mysql_models.py +++ b/app/models/grafana/mysql_models.py @@ -11,7 +11,7 @@ class TLS(BaseModel): class MysqlInput(BaseModel): name:str = "MySQL" - url:str = " localhost:3306" + url:str = "localhost:3306" user:str = "grafana" editable: bool = True database:str = "grafana" diff --git a/app/models/grafana/postgresql_models.py b/app/models/grafana/postgresql_models.py index 0850e700..54bf015c 100644 --- a/app/models/grafana/postgresql_models.py +++ b/app/models/grafana/postgresql_models.py @@ -10,7 +10,7 @@ class PostgresInput(BaseModel): user:str = "grafana" editable: bool = True database:str = "grafana" - sslmode:str = "disable" + sslmode:str = "'disable'" password:str = "Password!" maxOpenConns:int = 100 maxIdleConns:int = 100 diff --git a/app/models/grafana/prometheus_models.py b/app/models/grafana/prometheus_models.py index 3a04eedb..69ad98df 100644 --- a/app/models/grafana/prometheus_models.py +++ b/app/models/grafana/prometheus_models.py @@ -1,10 +1,8 @@ from typing import List, Optional -from pydantic import BaseModel, validator, ValidationError +from pydantic import BaseModel, validator, ValidationError,field_validator + -class TraceID(BaseModel): - datasourceUid:str = "my_jaeger_uid" - name:str = "traceID" class PrometheusInput(BaseModel): name:str = "Prometheus" url:str = "http://localhost:9090" @@ -13,7 +11,7 @@ class PrometheusInput(BaseModel): manageAlerts:bool = True prometheusType:str = "Prometheus" prometheusVersion:str = "2.44.0" - cacheLevel:str = 'High' + cacheLevel:str = "High" disableRecordingRules:bool = False incrementalQueryOverlapWindow:str = "10m" - trace_id: TraceID \ No newline at end of file + \ No newline at end of file diff --git a/app/routes/ansible.py b/app/routes/ansible.py index 367f543c..78c6fe72 100644 --- a/app/routes/ansible.py +++ b/app/routes/ansible.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.routes.utils import add_files_to_folder from app.models import (AnsibleInstallNginx,AnsibleInstallDocker,Output,AnsibleInstallKuber) diff --git a/app/routes/helm.py b/app/routes/helm.py index dde13d44..aff50423 100644 --- a/app/routes/helm.py +++ b/app/routes/helm.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.models import (HelmTemplateGeneration,Output) from app.prompt_generators import (helm_template_generator) import os diff --git a/app/routes/jcasc.py b/app/routes/jcasc.py index 5ad85ab6..937892d4 100644 --- a/app/routes/jcasc.py +++ b/app/routes/jcasc.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.models import (Jcasc,Output) from app.template_generators.jenkins.jcasc import jcasc_template_generator import os diff --git a/app/routes/terraform.py b/app/routes/terraform.py index 20ebbe03..19aeaf0a 100644 --- a/app/routes/terraform.py +++ b/app/routes/terraform.py @@ -1,7 +1,7 @@ from app.app_instance import app from app.gpt_services import gpt_service from app.services import ( - write_installation, + edit_directory_generator,execute_pythonfile) from app.models import (IaCBasicInput, IaCBugfixInput, diff --git a/app/services.py b/app/services.py index c7e3164a..b21e7304 100644 --- a/app/services.py +++ b/app/services.py @@ -1,37 +1,8 @@ -from .utils import save_to_mongo + import os import shutil from fastapi import HTTPException -def write_basic(request,output): - - data = { - 'question':request.question, - 'output':output - } - - save_to_mongo(data, index='question', collection = 'qa') - - -def write_bugfix(request,output): - data = { - 'bug_description':request.bug_description, - 'service':request.service, - 'output':output - } - - save_to_mongo(data, index=['bug_description','service'], collection = 'bugfix') - - -def write_installation(request,output): - - data = { - 'os':request.os, - 'service':request.service, - 'output':output - } - - save_to_mongo(data, index=['os','service'], collection = 'installation') def edit_directory_generator(gen_file,python_code): diff --git a/app/template_generators/grafana_data_sources/mimir.py b/app/template_generators/grafana_data_sources/mimir.py index 64122a10..aa83aac8 100644 --- a/app/template_generators/grafana_data_sources/mimir.py +++ b/app/template_generators/grafana_data_sources/mimir.py @@ -2,35 +2,67 @@ import os def mimir_template(input): - - json_template = { - "apiVersion": 1, - "datasources": [ - { - "name": input.name, - "uid": input.uid, - "type": "prometheus", - "access": "proxy", - "orgId": 1, - "url": input.url, - "editable": input.editable, - "version": 1, - "jsonData": { - "httpHeaderName1": input.httpHeaderName1, - "alertmanagerUid": input.alertmanagerUid - }, - "secureJsonData": { - "httpHeaderValue1": input.httpHeaderValue1 + if input.multi_tenancy is not None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "prometheus", + "access": "proxy", + "orgId": 1, + "url": input.url, + "editable": input.editable, + "version": 1, + "jsonData": { + "httpHeaderName1": input.httpHeaderName1, + "alertmanagerUid": input.alertmanagerUid + }, + "secureJsonData": { + "httpHeaderValue1": input.multi_tenancy.tenant_name + } + } + ] } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mimir.yml') + + file=open("app/media/MyGrafana/mimir.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "prometheus", + "access": "proxy", + "orgId": 1, + "url": input.url, + "editable": input.editable, + "version": 1, + "jsonData": { + "httpHeaderName1": input.httpHeaderName1, + "alertmanagerUid": input.alertmanagerUid + }, + + } + ] } - ] - } - dir = "app/media/MyGrafana" - os.makedirs(dir) - os.path.join(dir, 'mimir.yml') - - file=open("app/media/MyGrafana/mimir.yml","w") - yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mimir.yml') + + file=open("app/media/MyGrafana/mimir.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/prometheus.py b/app/template_generators/grafana_data_sources/prometheus.py index ef8afa18..580f09ff 100644 --- a/app/template_generators/grafana_data_sources/prometheus.py +++ b/app/template_generators/grafana_data_sources/prometheus.py @@ -1,7 +1,11 @@ import yaml import os - +from ruamel.yaml import YAML +from ruamel.yaml.scalarstring import ScalarString +import re def pormetheus_template(input): + + json_template = { "apiVersion": 1, "datasources": [ @@ -20,12 +24,7 @@ def pormetheus_template(input): "cacheLevel": input.cacheLevel, "disableRecordingRules": input.disableRecordingRules, "incrementalQueryOverlapWindow": input.incrementalQueryOverlapWindow, - "exemplarTraceIdDestinations": [ - { - "datasourceUid": input.trace_id.datasourceUid, - "name": input.trace_id.name - } - ] + } } ] @@ -34,8 +33,9 @@ def pormetheus_template(input): dir = "app/media/MyGrafana" os.makedirs(dir) - os.path.join(dir, 'prometheus.yml') - + os.path.join(dir, 'prometheus.yml') file=open("app/media/MyGrafana/prometheus.yml","w") yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + \ No newline at end of file diff --git a/app/utils.py b/app/utils.py index 39dfdb8e..8b137891 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,38 +1 @@ -import os -from fastapi import HTTPException -from pymongo import MongoClient,ASCENDING,errors - -def get_mongo_client(): - - client = MongoClient(host=os.environ.get('MONGO_HOST'), - port=int(os.environ.get('MONGO_PORT')), - username=os.environ.get('MONGO_INITDB_ROOT_USERNAME'), - password=os.environ.get('MONGO_INITDB_ROOT_PASSWORD')) - - return client - - - -def get_mongo_collection(col:str): - - client = get_mongo_client() - col = client[os.environ.get('MONGO_INITDB_DATABASE')][col] - return col - - -def save_to_mongo(data:dict,index:str,collection:str) -> None: - - try: - - col = get_mongo_collection(collection) - - col.create_index(index, unique=True) - - col.insert_one(data) - - except errors.DuplicateKeyError: - - pass - - diff --git a/requirements.txt b/requirements.txt index 5d269e0e..48e73996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ +ruamel.yaml<0.18.0 fastapi[standard]>=0.113.0,<0.114.0 pydantic>=2.7.0,<3.0.0 openai pytest -pymongo>=4.10.1 + From f171294f68209d344f8a2cc59fb8cff85d1cbc1f Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 17 Dec 2024 22:41:45 +0330 Subject: [PATCH 7/7] fix(mimir): edit mimir model --- app/media/MyGrafana/mimir.yml | 1 - app/models/grafana/mimir_models.py | 3 +-- app/template_generators/grafana_data_sources/mimir.py | 3 +-- app/template_generators/grafana_data_sources/prometheus.py | 7 ++++++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/media/MyGrafana/mimir.yml b/app/media/MyGrafana/mimir.yml index 01502a86..4ff9e8c4 100644 --- a/app/media/MyGrafana/mimir.yml +++ b/app/media/MyGrafana/mimir.yml @@ -9,5 +9,4 @@ datasources: editable: true version: 1 jsonData: - httpHeaderName1: X-Scope-OrgID alertmanagerUid: alertmanager diff --git a/app/models/grafana/mimir_models.py b/app/models/grafana/mimir_models.py index 13d0f616..7213961f 100644 --- a/app/models/grafana/mimir_models.py +++ b/app/models/grafana/mimir_models.py @@ -5,13 +5,12 @@ class MultiTenancy(BaseModel): tenant_name:str = "pods" - + httpHeaderName1:str = "X-Scope-OrgID" class MimirInput(BaseModel): name:str = "Mimir" uid:str = "mimir" url:str = "http://mimir-nginx.mimir.svc.cluster.local/prometheus" editable: bool = True - httpHeaderName1:str = "X-Scope-OrgID" alertmanagerUid:str = "alertmanager" multi_tenancy:Optional[MultiTenancy] diff --git a/app/template_generators/grafana_data_sources/mimir.py b/app/template_generators/grafana_data_sources/mimir.py index aa83aac8..1237dccd 100644 --- a/app/template_generators/grafana_data_sources/mimir.py +++ b/app/template_generators/grafana_data_sources/mimir.py @@ -16,7 +16,7 @@ def mimir_template(input): "editable": input.editable, "version": 1, "jsonData": { - "httpHeaderName1": input.httpHeaderName1, + "httpHeaderName1": input.multi_tenancy.httpHeaderName1, "alertmanagerUid": input.alertmanagerUid }, "secureJsonData": { @@ -48,7 +48,6 @@ def mimir_template(input): "editable": input.editable, "version": 1, "jsonData": { - "httpHeaderName1": input.httpHeaderName1, "alertmanagerUid": input.alertmanagerUid }, diff --git a/app/template_generators/grafana_data_sources/prometheus.py b/app/template_generators/grafana_data_sources/prometheus.py index 580f09ff..dcff822a 100644 --- a/app/template_generators/grafana_data_sources/prometheus.py +++ b/app/template_generators/grafana_data_sources/prometheus.py @@ -1,8 +1,13 @@ import yaml import os -from ruamel.yaml import YAML +import ruamel.yaml from ruamel.yaml.scalarstring import ScalarString import re + +class SingleQuotedScalarString(ScalarString): + def __new__(cls, value): + return ScalarString.__new__(cls, value) + def pormetheus_template(input):