feat: support K8S application logs (#97)

CONTRIBUTING.md

@@ -30,6 +30,8 @@ Start `minikube`:
 minikube start --insecure-registry "host.docker.internal:5000"
 ```
 
+* Note that if the minikube machine already exists, you will need to delete it first with `minikube delete` - otherwise the `--insecure-registry` parameter will be ignored (more details [here](https://stackoverflow.com/a/53937716))
+
 Start a local Docker registry:
 
 ```sh
@@ -46,15 +48,15 @@ $ curl localhost:5000/v2/_catalog -v
 > Host: localhost:5000
 > User-Agent: curl/7.77.0
 > Accept: */*
-> 
+>
 * Mark bundle as not supporting multiuse
 < HTTP/1.1 200 OK
 < Content-Type: application/json; charset=utf-8
 < Docker-Distribution-Api-Version: registry/2.0
 < X-Content-Type-Options: nosniff
 < Date: Fri, 20 Jan 2023 08:10:32 GMT
 < Content-Length: 20
-< 
+<
 {"repositories":[]}
 * Connection #0 to host localhost left intact
 ```
@@ -100,7 +102,7 @@ Changing the target Lumigo backend can be done with a [`patchStrategicMerge`](ht
 echo -n "apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: lumigo-controller-manager
+  name: lumigo-lumigo-operator-controller-manager
 spec:
   template:
     spec:
@@ -109,6 +111,8 @@ spec:
         env:
         - name: LUMIGO_ENDPOINT
           value: \"https://my.lumigo.endpoint\" # Replace this!
+        - name: LUMIGO_LOGS_ENDPOINT
+          value: \"https://my.lumigo.endpoint\" # Replace this!
 " > lumigo-endpoint.patch.yaml
 kubectl patch --patch-file lumigo-endpoint.patch.yaml --type strategic -n lumigo-system --filename=lumigo-endpoint.patch.yaml
 ```
@@ -123,8 +127,8 @@ If you see the following, it's likely because and Mac OS has [squatted over port
 docker push host.docker.internal:5000/controller
 Using default tag: latest
 The push refers to repository [host.docker.internal:5000/controller]
-377b701db379: Preparing 
-fba4381f2bb7: Preparing 
+377b701db379: Preparing
+fba4381f2bb7: Preparing
 error parsing HTTP 403 response body: unexpected end of JSON input: ""
 ```
 

Makefile

@@ -73,7 +73,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	(cd ./controller/src && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOCMD) test . -coverprofile cover.out )
+	(cd ./controller/src && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOCMD) test ./... -coverprofile cover.out )
 
 ##@ Build
 

README.md

@@ -72,9 +72,9 @@ The Lumigo Kubernetes operator allows you to set a human-readable name using the
 You can check which version of the Lumigo Kubernetes operator you have deployed in your cluster as follows:
 
 ```sh
-$ helm ls -A      
+$ helm ls -A
 NAME  	NAMESPACE    	REVISION	UPDATED                              	STATUS  	CHART             	APP VERSION
-lumigo	lumigo-system	2       	2023-07-10 09:20:04.233825 +0200 CEST	deployed	lumigo-operator-13	13         
+lumigo	lumigo-system	2       	2023-07-10 09:20:04.233825 +0200 CEST	deployed	lumigo-operator-13	13
 ```
 
 The Lumigo Kubernetes operator is reported as `APP VERSION`.
@@ -160,6 +160,26 @@ Status:
     UID:               93d6d809-ac2a-43a9-bc07-f0d4e314efcc
 ```
 
+#### Logging support
+
+The Lumigo Kubernetes operator can automatically forward logs emitted by traced pods to [Lumigo's log-management solution](https://lumigo.io/lp/log-management/), supporting several logging providers (currently `logging` for Python apps, `Winston` and `Bunyan` for Node.js apps).
+Enabling log forwarding is done by adding the `spec.logging.enabled` field to the `Lumigo` resource:
+
+```yaml
+apiVersion: operator.lumigo.io/v1alpha1
+kind: Lumigo
+metadata:
+  labels:
+    app.kubernetes.io/name: lumigo
+    app.kubernetes.io/instance: lumigo
+    app.kubernetes.io/part-of: lumigo-operator
+  name: lumigo
+spec:
+  lumigoToken: ... # same token used for tracing
+  logging:
+    enabled: true # enables log forwarding for pods with tracing injected
+```
+
 #### Opting out for specific resources
 
 To prevent the Lumigo Kubernetes operator from injecting tracing to pods managed by some resource in a namespace that contains a `Lumigo` resource, add the `lumigo.auto-trace` label set to `false`:

charts/lumigo-operator/templates/controller-deployment-and-webhooks.yaml

@@ -231,6 +231,8 @@ spec:
           value: "{{ .Values.debug.enabled | default false }}"
         - name: LUMIGO_ENDPOINT
           value: "{{ .Values.endpoint.otlp.url }}"
+        - name: LUMIGO_LOGS_ENDPOINT
+          value: "{{ .Values.endpoint.otlp.logs_url }}"
         - name: LUMIGO_OPERATOR_VERSION
           value: "{{ $lumigoOperatorVersion }}"
         - name: LUMIGO_OPERATOR_DEPLOYMENT_METHOD

charts/lumigo-operator/templates/lumigo-crd.yaml

@@ -102,6 +102,17 @@ spec:
                 required:
                 - injection
                 type: object
+              logging:
+                description: 'LoggingSpec specifies if logging should be set up by the operator'
+                properties:
+                  enabled:
+                    description: Whether Daemonsets, Deployments, ReplicaSets,
+                      StatefulSets, CronJobs and Jobs that are created or updated
+                      after the creation of the Lumigo resource and are injected will
+                      have their logs sent to Lumigo.
+                      If unspecified, defaults to `false`
+                    type: boolean
+                type: object
             type: object
           status:
             description: LumigoStatus defines the observed state of Lumigo

charts/lumigo-operator/values.yaml

@@ -61,4 +61,5 @@ metricsService:
   type: ClusterIP
 endpoint:
   otlp:
-    url: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+    url: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+    logs_url: https://ga-otlp.lumigo-tracer-edge.golumigo.com

config/crd/bases/operator.lumigo.io_lumigoes.yaml

@@ -102,6 +102,17 @@ spec:
                 required:
                 - injection
                 type: object
+              logging:
+                description: 'LoggingSpec specifies if logging should be set up by the operator'
+                properties:
+                  enabled:
+                    description: Whether Daemonsets, Deployments, ReplicaSets,
+                      StatefulSets, CronJobs and Jobs that are created or updated
+                      after the creation of the Lumigo resource and are injected will
+                      have their logs sent to Lumigo.
+                      If unspecified, defaults to `false`
+                    type: boolean
+                type: object
             type: object
           status:
             description: LumigoStatus defines the observed state of Lumigo

config/manager/manager.yaml

@@ -131,6 +131,8 @@ spec:
               value: Kustomize
             - name: LUMIGO_ENDPOINT
               value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+            - name: LUMIGO_LOGS_ENDPOINT
+              value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
           ports:
             - containerPort: 4318
               name: otlphttp

controller/src/api/v1alpha1/lumigo_types.go

@@ -54,6 +54,7 @@ type LumigoSpec struct {
 	// https://docs.lumigo.io/docs/lumigo-tokens
 	LumigoToken    Credentials        `json:"lumigoToken,omitempty"`
 	Tracing        TracingSpec        `json:"tracing,omitempty"`
+	Logging				 LoggingSpec        `json:"logging,omitempty"`
 	Infrastructure InfrastructureSpec `json:"infrastructure,omitempty"`
 }
 
@@ -77,6 +78,14 @@ type TracingSpec struct {
 	Injection InjectionSpec `json:"injection"`
 }
 
+type LoggingSpec struct {
+	// Whether Daemonsets, Deployments, ReplicaSets, StatefulSets, CronJobs and Jobs
+	// that are created or updated after the creation of the Lumigo resource have their logs sent to Lumigo.
+	// If unspecified, defaults to `false`.
+	// +kubebuilder:validation:Optional
+	Enabled *bool `json:"enabled"` // Using a pointer to support cases where the value is not set (and it counts as disabled)
+}
+
 type InjectionSpec struct {
 	// Whether Daemonsets, Deployments, ReplicaSets, StatefulSets, CronJobs and Jobs
 	// that are created or updated after the creation of the Lumigo resource be injected.

controller/src/api/v1alpha1/lumigo_webhook_suite_test.go

@@ -61,8 +61,8 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
 			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
 		},

controller/src/controllers/lumigo_controller.go

@@ -77,6 +77,7 @@ type LumigoReconciler struct {
 	LumigoOperatorVersion                     string
 	LumigoInjectorImage                       string
 	TelemetryProxyOtlpServiceUrl              string
+	TelemetryProxyOtlpLogsServiceUrl          string
 	TelemetryProxyNamespaceConfigurationsPath string
 }
 
@@ -489,7 +490,7 @@ func (r *LumigoReconciler) updateStatusIfNeeded(ctx context.Context, logger logr
 }
 
 func (r *LumigoReconciler) injectLumigoIntoResources(ctx context.Context, lumigo *operatorv1alpha1.Lumigo, log *logr.Logger) error {
-	mutator, err := mutation.NewMutator(log, &lumigo.Spec.LumigoToken, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl)
+	mutator, err := mutation.NewMutator(log, &lumigo.Spec, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl, r.TelemetryProxyOtlpLogsServiceUrl)
 	if err != nil {
 		return fmt.Errorf("cannot instantiate mutator: %w", err)
 	}
@@ -688,7 +689,8 @@ func (r *LumigoReconciler) injectLumigoIntoResources(ctx context.Context, lumigo
 
 func (r *LumigoReconciler) removeLumigoFromResources(ctx context.Context, lumigo *operatorv1alpha1.Lumigo, log *logr.Logger) error {
 	namespace := lumigo.Namespace
-	mutator, err := mutation.NewMutator(log, nil, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl)
+
+	mutator, err := mutation.NewMutator(log, nil, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl, r.TelemetryProxyOtlpLogsServiceUrl)
 	if err != nil {
 		return fmt.Errorf("cannot instantiate mutator: %w", err)
 	}

controller/src/controllers/lumigo_controller_suite_test.go

@@ -81,7 +81,7 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: true,
 	}
 
@@ -197,15 +197,14 @@ var _ = Context("Lumigo controller", func() {
 	})
 
 	Context("with one Lumigo instance", func() {
-
 		It("has an error if the referenced secret does not exist", func() {
 			lumigoName := "lumigo"
 			lumigo := newLumigo(namespaceName, lumigoName, operatorv1alpha1.Credentials{
 				SecretRef: operatorv1alpha1.KubernetesSecretRef{
 					Name: "lumigo-credentials",
 					Key:  "token",
 				},
-			}, true, true, true)
+			}, true, true, true, true)
 			Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 			By("the Lumigo instance goes in an erroneous state", func() {
@@ -253,7 +252,7 @@ var _ = Context("Lumigo controller", func() {
 					Name: "lumigo-credentials",
 					Key:  expectedTokenKey,
 				},
-			}, true, true, true)
+			}, true, true, true, true)
 			Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 			By("the Lumigo instance goes in an erroneous state", func() {
@@ -333,7 +332,7 @@ var _ = Context("Lumigo controller", func() {
 					Name: "lumigo-credentials",
 					Key:  expectedTokenKey,
 				},
-			}, true, true, true)
+			}, true, true, true, true)
 			Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 			By("the Lumigo instance goes in an erroneous state", func() {
@@ -434,7 +433,7 @@ var _ = Context("Lumigo controller", func() {
 							Name: lumigoSecretName,
 							Key:  expectedTokenKey,
 						},
-					}, true, false, false)
+					}, true, false, false, true)
 					g.Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 				}, defaultTimeout, defaultInterval).Should(Succeed())
 			})
@@ -453,7 +452,7 @@ var _ = Context("Lumigo controller", func() {
 					Name:      deploymentName,
 				}, deployment)).To(Succeed())
 
-				Expect(deployment).NotTo(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl))
+				Expect(deployment).NotTo(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl, false))
 			})
 
 		})
@@ -517,7 +516,7 @@ var _ = Context("Lumigo controller", func() {
 						Name: lumigoSecretName,
 						Key:  expectedTokenKey,
 					},
-				}, true, true, false)
+				}, true, true, false, false)
 				Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 				Eventually(func(g Gomega) {
@@ -537,7 +536,7 @@ var _ = Context("Lumigo controller", func() {
 						Name:      deploymentName,
 					}, deployment)).To(Succeed())
 
-					g.Expect(deployment).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl))
+					g.Expect(deployment).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl, false))
 					g.Expect(currentVersionOf(lumigo, g)).To(BeActive())
 					g.Expect(currentVersionOf(lumigo, g)).To(HaveInstrumentedObjectReferenceFor(deployment))
 				}, defaultTimeout, defaultInterval).Should(Succeed())
@@ -565,7 +564,7 @@ var _ = Context("Lumigo controller", func() {
 					Name:      deploymentName,
 				}, deployment)).To(Succeed())
 
-				Expect(deployment).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl))
+				Expect(deployment).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl, false))
 			})
 		})
 
@@ -628,7 +627,7 @@ var _ = Context("Lumigo controller", func() {
 						Name: lumigoSecretName,
 						Key:  expectedTokenKey,
 					},
-				}, true, true, true)
+				}, true, true, true, false)
 				Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 				Eventually(func(g Gomega) {
@@ -644,7 +643,7 @@ var _ = Context("Lumigo controller", func() {
 					Name:      deploymentName,
 				}, deploymentAfter)).To(Succeed())
 
-				Expect(deploymentAfter).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl))
+				Expect(deploymentAfter).To(mutation.BeInstrumentedWithLumigo(lumigoOperatorVersion, lumigoInjectorImage, telemetryProxyOtlpServiceUrl, false))
 			})
 
 			By("Deleting the Lumigo resource", func() {
@@ -710,7 +709,7 @@ var _ = Context("Lumigo controller", func() {
 						Name: lumigoSecretName,
 						Key:  expectedTokenKey,
 					},
-				}, true, true, false)
+				}, true, true, false, true)
 				Expect(k8sClient.Create(ctx, lumigo)).Should(Succeed())
 
 				Eventually(func(g Gomega) {
@@ -763,15 +762,15 @@ var _ = Context("Lumigo controller", func() {
 				},
 			}
 
-			lumigo1 := newLumigo(namespaceName, "lumigo1", lumigoToken, true, true, true)
+			lumigo1 := newLumigo(namespaceName, "lumigo1", lumigoToken, true, true, true, true)
 			Expect(k8sClient.Create(ctx, lumigo1)).Should(Succeed())
 			Eventually(func(g Gomega) {
 				g.Expect(currentVersionOf(lumigo1, g)).To(BeActive())
 			}, defaultTimeout, defaultInterval).Should(Succeed())
 
 			Expect(telemetryProxyNamespacesFile).To(BeMonitoringNamespace(namespaceName))
 
-			lumigo2 := newLumigo(namespaceName, "lumigo2", lumigoToken, true, true, true)
+			lumigo2 := newLumigo(namespaceName, "lumigo2", lumigoToken, true, true, true, true)
 			By("adding a second Lumigo in the namespace", func() {
 				Expect(k8sClient.Create(ctx, lumigo2)).Should(Succeed())
 
@@ -803,7 +802,7 @@ var _ = Context("Lumigo controller", func() {
 
 })
 
-func newLumigo(namespace string, name string, lumigoToken operatorv1alpha1.Credentials, injectionEnabled bool, injectLumigoIntoExistingResourcesOnCreation bool, removeLumigoFromResourcesOnDeletion bool) *operatorv1alpha1.Lumigo {
+func newLumigo(namespace string, name string, lumigoToken operatorv1alpha1.Credentials, injectionEnabled bool, injectLumigoIntoExistingResourcesOnCreation bool, removeLumigoFromResourcesOnDeletion bool, loggingEnabled bool) *operatorv1alpha1.Lumigo {
 	return &operatorv1alpha1.Lumigo{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
@@ -819,6 +818,9 @@ func newLumigo(namespace string, name string, lumigoToken operatorv1alpha1.Crede
 					RemoveLumigoFromResourcesOnDeletion:         &removeLumigoFromResourcesOnDeletion,
 				},
 			},
+			Logging: operatorv1alpha1.LoggingSpec{
+				Enabled: &loggingEnabled,
+			},
 		},
 	}
 }