diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index feb4e9b..8ba3765 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,15 +8,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: volesen/setup-skaffold@v1.1 - with: - version: v1.38.0 - - uses: extractions/setup-just@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - uses: bahmutov/npm-install@v1 - run: npx semantic-release env: diff --git a/.helmignore b/.helmignore index 6ac8534..5d83e4b 100644 --- a/.helmignore +++ b/.helmignore @@ -29,16 +29,16 @@ venv *.py *.txt .release-it.json -*.md +CONTRIBUTING.md skaffold.yaml values-lint.yaml node_modules -README.adoc *.sh package-lock.json +package.json *.puml .editorconfig .dockerignore test/ -cmak2zk/ +justfile diff --git a/.releaserc.yaml b/.releaserc.yaml index 40e0dfb..381dd89 100644 --- a/.releaserc.yaml +++ b/.releaserc.yaml @@ -20,7 +20,3 @@ plugins: - - '@eshepelyuk/semantic-release-helm-oci' - registry: oci://ghcr.io/eshepelyuk/helm - skipAppVersion: true - - - - "@semantic-release/exec" - - publishCmd: "just publish ${nextRelease.version}" diff --git a/README.md b/README.md index e06b935..5fd040a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ - - # CMAK (prev. Kafka Manager) for Kubernetes [![Current](https://img.shields.io/github/v/tag/eshepelyuk/cmak-operator?logo=github&sort=semver&style=for-the-badge&label=current)](https://github.com/eshepelyuk/cmak-operator/releases/latest) @@ -128,42 +126,6 @@ cmak: 1. applied only to `cluster-stage`. 1. applied only to `cluster-prod`. -## Standalone cmak2zk tool - -`cmak2zk` was developed as a part of CMAK operator and actively used by the operator itself. -But the same time this tool could be used on its own outside of Helm charts and Kubernetes. - -Its purpose is to take Kafka cluster configuration for CMAK in YAML format -and populate CMAK compatible config in Zookeeper. -This allows to avoid manual configuration of CMAK and provides better possibilities -to use CMAK in declarative configuration or GitOps based flows. - -`cmak2zk` is distributed as [docker image](https://github.com/users/eshepelyuk/packages/container/package/cmak2zk). - -To check out available options, run the image without parameters. - -```sh -docker run ghcr.io/eshepelyuk/dckr/cmak2zk:latest -``` - -Example `docker-compose` and Kafka cluster configuration are located at -[cmak2zk/examples](https://github.com/eshepelyuk/cmak-operator/tree/master/cmak2zk/examples) directory. -One could run them using commands below. - -```sh -curl -sLo clusters.yaml \ - https://raw.githubusercontent.com/eshepelyuk/cmak-operator/master/cmak2zk/examples/clusters.yaml - -curl -sLo docker-compose-cmak2zk.yaml \ - https://raw.githubusercontent.com/eshepelyuk/cmak-operator/master/cmak2zk/examples/docker-compose-cmak2zk.yaml - -docker-compose -f docker-compose-cmak2zk.yaml up -``` - -Wait for some time until components are stabilizing, it may take up to 5 mins. -Then, open your browser at http://localhost:9000. -There should be two pre-configured clusters, pointing to the same Kafka instance, running in Docker. - ## Alternatives [AKHQ](https://akhq.io/) project seems to be the most active open source tool diff --git a/cmak2zk/Dockerfile b/cmak2zk/Dockerfile deleted file mode 100644 index f440a6b..0000000 --- a/cmak2zk/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.8-alpine3.12 - -COPY *.py /app/bin/ - -RUN set -x && pip install click~=7.0 kazoo~=2.8 ruamel.yaml~=0.16 jsonmerge~=1.8 - -ENTRYPOINT ["python3", "/app/bin/cmak2zk.py"] -CMD ["--help"] - -WORKDIR /app/ diff --git a/cmak2zk/cmak2zk.py b/cmak2zk/cmak2zk.py deleted file mode 100644 index 0edc14f..0000000 --- a/cmak2zk/cmak2zk.py +++ /dev/null @@ -1,63 +0,0 @@ -import click -import logging -from kazoo.client import KazooClient -from pathlib import Path -from hashlib import md5 -from ruamel.yaml import YAML -from jsonmerge import merge -import json - -FORMAT = '%(asctime)-15s %(message)s' -logging.basicConfig(format=FORMAT, level=logging.INFO) - -ZK_ROOT="/kafka-manager/configs" - -yaml = YAML(typ='safe') - - -@click.command() -@click.option('--overwrite-zk/--no-overwrite-zk', "over_zk", default=True, help="should existing ZK data be rewritten on md5 difference") -@click.argument("zk_url", type=str) -@click.argument('yaml_cfg', type=click.File()) -def cmak2zk(over_zk, zk_url, yaml_cfg): - """ - Populates Zookeeper at ZK_URL with Kafka cluster configuration - in CMAK compatible format from YAML_CFG configuration file. - - ZK_URL - myzk1:2181,myzk2:2181, etc. - - \b - YAML_CFG - Format is equal to CMAK operator Helm chart values. - The only section used is - https://artifacthub.io/packages/helm/cmak-operator/cmak-operator?modal=values-schema&path=cmak. - - """ - cmak_cfg = yaml.load(yaml_cfg)['cmak'] - common_cfg = cmak_cfg['clustersCommon'] - - zk = KazooClient(hosts=zk_url) - zk.start() - - for cl in cmak_cfg['clusters']: - cl = merge(common_cfg, cl) - dst = f"{ZK_ROOT}/{cl['name']}" - json_b = json.dumps(cl, separators=(',', ':')).encode() - - if zk.exists(dst): - file_md5 = md5(json_b).hexdigest() - - zk_b, stat = zk.get(dst) - zk_md5 = md5(zk_b).hexdigest() - logging.info(f"md5 of {dst}: {zk_md5}, md5 of {cl['name']}: {file_md5}") - - if zk_md5 != file_md5 and over_zk is True: - zk.set(dst, json_b) - logging.info(f"Overwritten {dst} from {yaml_cfg.name}") - else: - zk.create(dst, json_b, makepath=True) - logging.info(f"Created {dst} from {yaml_cfg.name}") - - zk.stop() - -if __name__ == "__main__": - cmak2zk() diff --git a/cmak2zk/examples/clusters.yaml b/cmak2zk/examples/clusters.yaml deleted file mode 100644 index 91ea496..0000000 --- a/cmak2zk/examples/clusters.yaml +++ /dev/null @@ -1,48 +0,0 @@ -cmak: - clustersCommon: - curatorConfig: - zkMaxRetry: 100 - baseSleepTimeMs: 100 - maxSleepTimeMs: 1000 - enabled: true - kafkaVersion: "2.4.0" - jmxEnabled: false - jmxUser: null - jmxPass: null - jmxSsl: false - pollConsumers: true - filterConsumers: false - logkafkaEnabled: false - activeOffsetCacheEnabled: true - displaySizeEnabled: false - tuning: - brokerViewUpdatePeriodSeconds: 30 - clusterManagerThreadPoolSize: 10 - clusterManagerThreadPoolQueueSize: 100 - kafkaCommandThreadPoolSize: 10 - kafkaCommandThreadPoolQueueSize: 100 - logkafkaCommandThreadPoolSize: 10 - logkafkaCommandThreadPoolQueueSize: 100 - logkafkaUpdatePeriodSeconds: 30 - partitionOffsetCacheTimeoutSecs: 5 - brokerViewThreadPoolSize: 10 - brokerViewThreadPoolQueueSize: 1000 - offsetCacheThreadPoolSize: 10 - offsetCacheThreadPoolQueueSize: 1000 - kafkaAdminClientThreadPoolSize: 10 - kafkaAdminClientThreadPoolQueueSize: 1000 - kafkaManagedOffsetMetadataCheckMillis: 30000 - kafkaManagedOffsetGroupCacheSize: 1000000 - kafkaManagedOffsetGroupExpireDays: 7 - securityProtocol: PLAINTEXT - saslMechanism: null - jaasConfig: null - - clusters: - - name: test - curatorConfig: - zkConnect: zk:2181 - - name: test-disabled - enabled: false - curatorConfig: - zkConnect: zk:2181 diff --git a/cmak2zk/examples/docker-compose-cmak2zk.yaml b/cmak2zk/examples/docker-compose-cmak2zk.yaml deleted file mode 100644 index 56cf8a3..0000000 --- a/cmak2zk/examples/docker-compose-cmak2zk.yaml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.6' -services: - zk: - image: zookeeper:latest - restart: always - environment: - ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 - - cmak: - image: ghcr.io/eshepelyuk/dckr/cmak-3.0.0.5:latest - restart: always - ports: - - "9000:9000" - depends_on: - - "zk" - environment: - ZK_HOSTS: "zk:2181" - - cmak2zk: - image: ghcr.io/eshepelyuk/dckr/cmak2zk:latest - restart: on-failure - command: - - 'zk:2181' - - '/app/etc/clusters.yaml' - depends_on: - - "zk" - volumes: - - "${PWD}/clusters.yaml:/app/etc/clusters.yaml:ro" - -############################################################### -# This Kafka setup is only for demonstration purpose. # -############################################################### - kafka: - image: wurstmeister/kafka:2.12-2.4.1 - restart: always - depends_on: - - "zk" - environment: - KAFKA_ADVERTISED_HOST_NAME: "kafka" - KAFKA_CREATE_TOPICS: "test:1:1" - KAFKA_ZOOKEEPER_CONNECT: "zk:2181" -############################################################### diff --git a/justfile b/justfile index 466cf4c..c0bdf93 100644 --- a/justfile +++ b/justfile @@ -1,8 +1,7 @@ -skaffoldTags := "tags.json" export E2E_TEST := "default" default: - @just --list + @just --list test-lint: ./test/linter/test.sh @@ -62,14 +61,3 @@ test-e2e-sh: _chk-py # run single e2e test test-e2e: up test-e2e-sh -publish version: - #!/usr/bin/env bash - set -euxo pipefail - - skaffold build -t {{version}} --file-output={{skaffoldTags}} - - LATEST="$(jq -r .builds[0].imageName {{skaffoldTags}}):latest" - CURRENT="$(jq -r .builds[0].tag {{skaffoldTags}})" - - docker tag $CURRENT $LATEST - docker push $LATEST diff --git a/skaffold.yaml b/skaffold.yaml index 839eb27..a4be605 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -2,13 +2,6 @@ apiVersion: skaffold/v2beta28 kind: Config metadata: name: cmak-operator -build: - artifacts: - - image: ghcr.io/eshepelyuk/dckr/cmak2zk - context: ./cmak2zk - local: - push: true - useBuildkit: true deploy: helm: releases: @@ -21,7 +14,3 @@ deploy: - "test/e2e/{{.E2E_TEST}}/values.yaml" upgradeOnChange: false skipBuildDependencies: true - artifactOverrides: - reconcile.image: ghcr.io/eshepelyuk/dckr/cmak2zk - imageStrategy: - helm: {} diff --git a/values.yaml b/values.yaml index 9c9486f..87cc460 100644 --- a/values.yaml +++ b/values.yaml @@ -44,7 +44,7 @@ cmak: reconcile: image: repository: "ghcr.io/eshepelyuk/dckr/cmak2zk" - tag: "" # chart's version + tag: "1.0.0" pullPolicy: IfNotPresent schedule: "*/3 * * * *" overwriteZk: true @@ -64,7 +64,7 @@ reconcile: ui: image: repository: "ghcr.io/eshepelyuk/dckr/cmak-3.0.0.5" - tag: "0.1.2" + tag: "1.0.0" pullPolicy: IfNotPresent port: 9000 # additional command line arguments