Merge branch 'develop' into json-log-coll

SigNoz · Sep 11, 2024 · a2cb5d5 · a2cb5d5
2 parents 8b5e839 + d6b75d7
commit a2cb5d5
Show file tree

Hide file tree

Showing 948 changed files with 51,328 additions and 6,788 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -5,6 +5,6 @@
 /frontend/ @YounixM
 /frontend/src/container/MetricsApplication @srikanthccv
 /frontend/src/container/NewWidget/RightContainer/types.ts @srikanthccv
-/deploy/ @prashant-shahi
-/sample-apps/ @prashant-shahi
-.github @prashant-shahi
+/deploy/ @SigNoz/devops
+/sample-apps/ @SigNoz/devops
+.github @SigNoz/devops
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -8,6 +8,13 @@ on:
       - release/v*
 
 jobs:
+  check-no-ee-references:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run check
+      run: make check-no-ee-references
+
   build-frontend:
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -158,6 +158,7 @@ jobs:
           echo 'SENTRY_DSN="${{ secrets.SENTRY_DSN }}"' >> frontend/.env
           echo 'TUNNEL_URL="${{ secrets.TUNNEL_URL }}"' >> frontend/.env
           echo 'TUNNEL_DOMAIN="${{ secrets.TUNNEL_DOMAIN }}"' >> frontend/.env
+          echo 'POSTHOG_KEY="${{ secrets.POSTHOG_KEY }}"' >> frontend/.env
       - name: Install dependencies
         working-directory: frontend
         run: yarn install

diff --git a/.github/workflows/staging-deployment.yaml b/.github/workflows/staging-deployment.yaml
@@ -30,6 +30,7 @@ jobs:
           GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
           GCP_ZONE: ${{ secrets.GCP_ZONE }}
           GCP_INSTANCE: ${{ secrets.GCP_INSTANCE }}
+          CLOUDSDK_CORE_DISABLE_PROMPTS: 1
         run: |
           read -r -d '' COMMAND <<EOF || true
           echo "GITHUB_BRANCH: ${GITHUB_BRANCH}"
@@ -51,4 +52,4 @@ jobs:
           make build-frontend-amd64
           make run-testing
           EOF
-          gcloud compute ssh ${GCP_INSTANCE} --zone ${GCP_ZONE} --ssh-key-expire-after=15m --tunnel-through-iap --project ${GCP_PROJECT} --command "${COMMAND}"
+          gcloud beta compute ssh ${GCP_INSTANCE} --zone ${GCP_ZONE} --ssh-key-expire-after=15m --tunnel-through-iap --project ${GCP_PROJECT} --command "${COMMAND}"
diff --git a/.github/workflows/testing-deployment.yaml b/.github/workflows/testing-deployment.yaml
@@ -30,6 +30,7 @@ jobs:
           GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
           GCP_ZONE: ${{ secrets.GCP_ZONE }}
           GCP_INSTANCE: ${{ secrets.GCP_INSTANCE }}
+          CLOUDSDK_CORE_DISABLE_PROMPTS: 1
         run: |
           read -r -d '' COMMAND <<EOF || true
           echo "GITHUB_BRANCH: ${GITHUB_BRANCH}"
@@ -52,4 +53,4 @@ jobs:
           make build-frontend-amd64
           make run-testing
           EOF
-          gcloud compute ssh ${GCP_INSTANCE} --zone ${GCP_ZONE} --ssh-key-expire-after=15m --tunnel-through-iap --project ${GCP_PROJECT} --command "${COMMAND}"
+          gcloud beta compute ssh ${GCP_INSTANCE} --zone ${GCP_ZONE} --ssh-key-expire-after=15m --tunnel-through-iap --project ${GCP_PROJECT} --command "${COMMAND}"
diff --git a/.gitignore b/.gitignore
@@ -67,3 +67,6 @@ e2e/.auth
 # go
 vendor/
 **/main/**
+
+# git-town
+.git-branches.toml
diff --git a/.scripts/commentLinesForSetup.sh b/.scripts/commentLinesForSetup.sh
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -347,7 +347,7 @@ curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-
 ```bash
 kubectl -n sample-application run strzal --image=djbingham/curl \
   --restart='OnFailure' -i --tty --rm --command -- curl -X POST -F \
-  'locust_count=6' -F 'hatch_rate=2' http://locust-master:8089/swarm
+  'user_count=6' -F 'spawn_rate=2' http://locust-master:8089/swarm
 ```
 
 **5.1.3 To stop the load generation:**

diff --git a/Makefile b/Makefile
@@ -178,6 +178,15 @@ clear-swarm-ch:
 	@docker run --rm -v "$(PWD)/$(SWARM_DIRECTORY)/data:/pwd" busybox \
 	sh -c "cd /pwd && rm -rf clickhouse*/* zookeeper-*/*"
 
+check-no-ee-references:
+	@echo "Checking for 'ee' package references in 'pkg' directory..."
+	@if grep -R --include="*.go" '.*/ee/.*' pkg/; then \
+		echo "Error: Found references to 'ee' packages in 'pkg' directory"; \
+		exit 1; \
+	else \
+		echo "No references to 'ee' packages found in 'pkg' directory"; \
+	fi
+
 test:
 	go test ./pkg/query-service/app/metrics/...
 	go test ./pkg/query-service/cache/...
@@ -188,3 +197,4 @@ test:
 	go test ./pkg/query-service/tests/integration/...
 	go test ./pkg/query-service/rules/...
 	go test ./pkg/query-service/collectorsimulator/...
+	go test ./pkg/query-service/postprocess/...
diff --git a/README.md b/README.md
@@ -198,14 +198,14 @@ Not sure how to get started? Just ping us on `#contributing` in our [slack commu
 
 #### Frontend
 
-- [Palash Gupta](https://github.com/palashgdev)
 - [Yunus M](https://github.com/YounixM)
-- [Rajat Dabade](https://github.com/Rajat-Dabade)
+- [Vikrant Gupta](https://github.com/vikrantgupta25)
+- [Sagar Rajput](https://github.com/SagarRajput-7)
 
 #### DevOps
 
 - [Prashant Shahi](https://github.com/prashant-shahi)
-- [Dhawal Sanghvi](https://github.com/dhawal1248)
+- [Vibhu Pandey](https://github.com/grandwizard28)
 
 <br /><br />
 

diff --git a/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml b/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml
@@ -649,12 +649,12 @@
 
          See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables
       -->
-    <!--
+
     <macros>
         <shard>01</shard>
         <replica>example01-01-1</replica>
     </macros>
-    -->
+
 
 
     <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->

diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
@@ -146,7 +146,7 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.46.0
+    image: signoz/query-service:0.49.1
     command:
       [
         "-config=/root/config/prometheus.yml",
@@ -186,7 +186,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.46.0
+    image: signoz/frontend:0.48.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -199,7 +199,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.88.24
+    image: signoz/signoz-otel-collector:0.102.2
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -211,6 +211,7 @@ services:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}}
       - DOCKER_MULTI_NODE_CLUSTER=false
@@ -237,7 +238,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.88.24
+      image: signoz/signoz-schema-migrator:0.102.2
       deploy:
         restart_policy:
           condition: on-failure

diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
@@ -36,6 +36,7 @@ receivers:
       #   endpoint: 0.0.0.0:6832
   hostmetrics:
     collection_interval: 30s
+    root_path: /hostfs
     scrapers:
       cpu: {}
       load: {}

diff --git a/deploy/docker/clickhouse-setup/clickhouse-config.xml b/deploy/docker/clickhouse-setup/clickhouse-config.xml
@@ -649,12 +649,12 @@
 
          See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables
       -->
-    <!--
+
     <macros>
         <shard>01</shard>
         <replica>example01-01-1</replica>
     </macros>
-    -->
+
 
 
     <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->

diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml
@@ -66,7 +66,7 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.24}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -81,7 +81,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.88.24
+    image: signoz/signoz-otel-collector:0.102.2
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -93,6 +93,8 @@ services:
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
     ports:

diff --git a/deploy/docker/clickhouse-setup/docker-compose.testing.yaml b/deploy/docker/clickhouse-setup/docker-compose.testing.yaml
@@ -164,7 +164,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.46.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.49.1}
     container_name: signoz-query-service
     command:
       [
@@ -204,7 +204,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.46.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.49.1}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -216,7 +216,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.24}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -230,7 +230,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.24}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.2}
     container_name: signoz-otel-collector
     command:
       [
@@ -244,6 +244,7 @@ services:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
       - DOCKER_MULTI_NODE_CLUSTER=false

diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml
@@ -164,7 +164,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.46.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.49.1}
     container_name: signoz-query-service
     command:
       [
@@ -203,7 +203,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.46.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.49.1}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -215,7 +215,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.24}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -229,7 +229,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.24}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.2}
     container_name: signoz-otel-collector
     command:
       [
@@ -243,6 +243,7 @@ services:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
       - DOCKER_MULTI_NODE_CLUSTER=false

diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
@@ -36,6 +36,7 @@ receivers:
       #   endpoint: 0.0.0.0:6832
   hostmetrics:
     collection_interval: 30s
+    root_path: /hostfs
     scrapers:
       cpu: {}
       load: {}

diff --git a/deploy/docker/common/nginx-config.conf b/deploy/docker/common/nginx-config.conf
@@ -1,3 +1,8 @@
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
 server {
     listen       3301;
     server_name  _;
@@ -42,6 +47,14 @@ server {
         proxy_read_timeout 600s; 
     }
 
+    location /ws {
+        proxy_pass  http://query-service:8080/ws;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade "websocket";
+        proxy_set_header Connection "upgrade";
+        proxy_read_timeout 86400;
+    }
+
     # redirect server error pages to the static page /50x.html
     #
     error_page   500 502 503 504  /50x.html;

diff --git a/deploy/install.sh b/deploy/install.sh
@@ -389,7 +389,7 @@ trap bye EXIT
 
 URL="https://api.segment.io/v1/track"
 HEADER_1="Content-Type: application/json"
-HEADER_2="Authorization: Basic NEdtb2E0aXhKQVVIeDJCcEp4c2p3QTFiRWZud0VlUno6"
+HEADER_2="Authorization: Basic OWtScko3b1BDR1BFSkxGNlFqTVBMdDVibGpGaFJRQnI="
 
 send_event() {
     error=""

diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go
@@ -24,7 +24,6 @@ import (
 type APIHandlerOptions struct {
 	DataConnector                 interfaces.DataConnector
 	SkipConfig                    *basemodel.SkipConfig
-	PreferDelta                   bool
 	PreferSpanMetrics             bool
 	MaxIdleConns                  int
 	MaxOpenConns                  int
@@ -53,7 +52,6 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) {
 	baseHandler, err := baseapp.NewAPIHandler(baseapp.APIHandlerOpts{
 		Reader:                        opts.DataConnector,
 		SkipConfig:                    opts.SkipConfig,
-		PerferDelta:                   opts.PreferDelta,
 		PreferSpanMetrics:             opts.PreferSpanMetrics,
 		MaxIdleConns:                  opts.MaxIdleConns,
 		MaxOpenConns:                  opts.MaxOpenConns,

diff --git a/ee/query-service/app/api/dashboard.go b/ee/query-service/app/api/dashboard.go
@@ -1,7 +1,9 @@
 package api
 
 import (
+	"errors"
 	"net/http"
+	"strings"
 
 	"github.com/gorilla/mux"
 	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
@@ -29,6 +31,10 @@ func (ah *APIHandler) lockUnlockDashboard(w http.ResponseWriter, r *http.Request
 
 	// Get the dashboard UUID from the request
 	uuid := mux.Vars(r)["uuid"]
+	if strings.HasPrefix(uuid,"integration") {
+		RespondError(w, &model.ApiError{Typ: model.ErrorForbidden, Err: errors.New("dashboards created by integrations cannot be unlocked")}, "You are not authorized to lock/unlock this dashboard")
+		return 
+	}
 	dashboard, err := dashboards.GetDashboard(r.Context(), uuid)
 	if err != nil {
 		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, err.Error())