Skip to content

Commit

Permalink
Merge pull request #186 from nickytd/test-renaming-branch-c97ce2fa
Browse files Browse the repository at this point in the history
Loki to Vali migration
  • Loading branch information
nickytd authored May 4, 2023
2 parents 0d23a36 + 44a1672 commit bbfc3c5
Show file tree
Hide file tree
Showing 153 changed files with 874 additions and 771 deletions.
8 changes: 4 additions & 4 deletions .ci/pipeline_definitions
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ logging:
- linux/amd64
- linux/arm64
dockerimages:
fluent-bit-to-loki:
fluent-bit-to-vali:
registry: 'gcr-readwrite'
image: 'eu.gcr.io/gardener-project/gardener/fluent-bit-to-loki'
image: 'eu.gcr.io/gardener-project/gardener/fluent-bit-to-vali'
dockerfile: './Dockerfile'
target: fluent-bit-plugin
resource_labels:
Expand All @@ -27,9 +27,9 @@ logging:
integrity_requirement: 'none'
availability_requirement: 'none'
comment: no data is stored or processed by the installer
loki-curator:
vali-curator:
registry: 'gcr-readwrite'
image: 'eu.gcr.io/gardener-project/gardener/loki-curator'
image: 'eu.gcr.io/gardener-project/gardener/vali-curator'
dockerfile: './Dockerfile'
target: curator
resource_labels:
Expand Down
4 changes: 2 additions & 2 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ run:

skip-files:
- ".*\\.pb\\.go$"
- ".*/out_loki\\.go$"
- ".*/out_vali\\.go$"
- ".*/.*_test\\.go$"
- "tests/.*"

Expand Down Expand Up @@ -39,6 +39,6 @@ issues:
- linters:
- staticcheck
text: "SA1019:" # Excludes messages where deprecated variables are used
- path: cmd/out_loki\.go
- path: cmd/out_vali\.go
linters:
- deadcode
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ RUN make install EFFECTIVE_VERSION=$EFFECTIVE_VERSION GOARCH=$TARGETARCH
############# curator #############
FROM distroless-static AS curator

COPY --from=image-builder /go/bin/loki-curator /curator
COPY --from=image-builder /go/bin/vali-curator /curator

WORKDIR /
EXPOSE 2718
Expand Down
28 changes: 14 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
VERSION := $(shell cat VERSION)
REGISTRY := eu.gcr.io/gardener-project/gardener
FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY := $(REGISTRY)/fluent-bit-to-loki
LOKI_CURATOR_IMAGE_REPOSITORY := $(REGISTRY)/loki-curator
FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY := $(REGISTRY)/fluent-bit-to-vali
VALI_CURATOR_IMAGE_REPOSITORY := $(REGISTRY)/vali-curator
TELEGRAF_IMAGE_REPOSITORY := $(REGISTRY)/telegraf-iptables
TUNE2FS_IMAGE_REPOSITORY := $(REGISTRY)/tune2fs
EVENT_LOGGER_IMAGE_REPOSITORY := $(REGISTRY)/event-logger
Expand All @@ -26,12 +26,12 @@ GOARCH := amd64

.PHONY: plugin
plugin:
go build -mod=vendor -buildmode=c-shared -o build/out_loki.so ./cmd/fluent-bit-loki-plugin
go build -mod=vendor -buildmode=c-shared -o build/out_vali.so ./cmd/fluent-bit-vali-plugin

.PHONY: curator
curator:
CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) GO111MODULE=on \
go build -mod=vendor -o build/curator ./cmd/loki-curator
go build -mod=vendor -o build/curator ./cmd/vali-curator

.PHONY: event-logger
event-logger:
Expand All @@ -42,11 +42,11 @@ event-logger:
build: plugin

.PHONY: install
install: install-loki-curator install-event-logger
install: install-vali-curator install-event-logger

.PHONY: install-loki-curator
install-loki-curator:
@EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) ./hack/install.sh ./cmd/loki-curator
.PHONY: install-vali-curator
install-vali-curator:
@EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) ./hack/install.sh ./cmd/vali-curator

.PHONY: install-event-logger
install-event-logger:
Expand All @@ -58,18 +58,18 @@ install-copy:

.PHONY: docker-images
docker-images:
@docker build -t $(FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY):latest -f Dockerfile --target fluent-bit-plugin .
@docker build -t $(LOKI_CURATOR_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(LOKI_CURATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target curator .
@docker build -t $(FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY):latest -f Dockerfile --target fluent-bit-plugin .
@docker build -t $(VALI_CURATOR_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(VALI_CURATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target curator .
@docker build -t $(TELEGRAF_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(TELEGRAF_IMAGE_REPOSITORY):latest -f Dockerfile --target telegraf .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EVENT_LOGGER_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(EVENT_LOGGER_IMAGE_REPOSITORY):latest -f Dockerfile --target event-logger .
@docker build -t $(TUNE2FS_IMAGE_REPOSITORY):$(IMAGE_TAG) -t $(TUNE2FS_IMAGE_REPOSITORY):latest -f Dockerfile --target tune2fs .

.PHONY: docker-push
docker-push:
@if ! docker images $(FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
@gcloud docker -- push $(FLUENT_BIT_TO_LOKI_IMAGE_REPOSITORY):$(IMAGE_TAG)
@if ! docker images $(LOKI_CURATOR_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(LOKI_CURATOR_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
@gcloud docker -- push $(LOKI_CURATOR_IMAGE_REPOSITORY):$(IMAGE_TAG)
@if ! docker images $(FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
@gcloud docker -- push $(FLUENT_BIT_TO_VALI_IMAGE_REPOSITORY):$(IMAGE_TAG)
@if ! docker images $(VALI_CURATOR_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(VALI_CURATOR_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
@gcloud docker -- push $(VALI_CURATOR_IMAGE_REPOSITORY):$(IMAGE_TAG)
@if ! docker images $(TELEGRAF_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(TELEGRAF_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
@gcloud docker -- push $(TELEGRAF_IMAGE_REPOSITORY):$(IMAGE_TAG)
@if ! docker images $(EVENT_LOGGER_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(IMAGE_TAG); then echo "$(EVENT_LOGGER_IMAGE_REPOSITORY) version $(IMAGE_TAG) is not yet built. Please run 'make docker-images'"; false; fi
Expand Down
6 changes: 3 additions & 3 deletions NOTICE.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Notices for GardenerLoki output plugin
# Notices for GardenerVali output plugin

This plugin extends [Grafana/Loki](https://github.com/grafana/loki) [fluent-bit output plugin v1.6.0](https://github.com/grafana/loki/tree/v1.6.0/cmd/fluent-bit) which aims to forward log messages from fluent-bit to Loki. It is maintained by SAP.
This plugin extends [Credativ/Vali](https://github.com/credativ/vali) [fluent-bit output plugin v1.6.0](https://github.com/credativ/vali/tree/v1.6.0/cmd/fluent-bit) which aims to forward log messages from fluent-bit to Vali. It is maintained by SAP.

## Copyright

Expand All @@ -10,4 +10,4 @@ source code repository logs.

## Modifications

After coping the original plugin from [Grafana/Loki](https://github.com/grafana/loki/tree/v1.6.0/cmd/fluent-bit) the files, which was in one single directory, was splitted to different packages. Then a controller package was added. The controller use shared informer to watch for namespaces and process the CREATE, UPDATE, DELETE Events. The `loki.go` file was modified to implement a function for storing the new controller and to search for the dynamic host path in the currently processing log entry. The `config.go` was modified to parse additional properties needed for the new functionality. `loki.go` was split to `loki.go` and `utils.go` to separate methods from helper functions. `out_loki.go` was modified to initialize cluster kubernetes client and make a shared informer which will be passed to each controller.
After coping the original plugin from [Credativ/Vali](https://github.com/credativ/vali/tree/v1.6.0/cmd/fluent-bit) the files, which was in one single directory, was splitted to different packages. Then a controller package was added. The controller use shared informer to watch for namespaces and process the CREATE, UPDATE, DELETE Events. The `vali.go` file was modified to implement a function for storing the new controller and to search for the dynamic host path in the currently processing log entry. The `config.go` was modified to parse additional properties needed for the new functionality. `vali.go` was split to `vali.go` and `utils.go` to separate methods from helper functions. `out_vali.go` was modified to initialize cluster kubernetes client and make a shared informer which will be passed to each controller.
Loading

0 comments on commit bbfc3c5

Please sign in to comment.