From df66c094e29b55d92ef2ef6bce9b2ab14160e6ca Mon Sep 17 00:00:00 2001 From: d-kuro Date: Fri, 23 Jun 2023 11:45:01 +0900 Subject: [PATCH] Build fluent-bit container image. (#528) * Build fluent-bit container image. Signed-off-by: d-kuro * Remove BRANCH file Signed-off-by: d-kuro * Fix tag script. Signed-off-by: d-kuro * Use larger_runner_16core Signed-off-by: d-kuro * Fix container link Signed-off-by: d-kuro * Add debug value Signed-off-by: d-kuro * Fix review feedback issues. Signed-off-by: d-kuro Co-authored-by: masa213f --------- Signed-off-by: d-kuro Co-authored-by: masa213f --- .../workflows/build-fluent-bit-container.yaml | 47 ++++++++++++ .github/workflows/ci-e2e.yaml | 8 +- containers/fluent-bit/.dockerignore | 1 + containers/fluent-bit/Dockerfile | 74 +++++++++++++++++++ containers/fluent-bit/README.md | 11 +++ containers/fluent-bit/TAG | 1 + containers/tag_exists | 36 +++++++++ 7 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-fluent-bit-container.yaml create mode 100644 containers/fluent-bit/.dockerignore create mode 100644 containers/fluent-bit/Dockerfile create mode 100644 containers/fluent-bit/README.md create mode 100644 containers/fluent-bit/TAG create mode 100755 containers/tag_exists diff --git a/.github/workflows/build-fluent-bit-container.yaml b/.github/workflows/build-fluent-bit-container.yaml new file mode 100644 index 000000000..c2043b0d4 --- /dev/null +++ b/.github/workflows/build-fluent-bit-container.yaml @@ -0,0 +1,47 @@ +name: "Build Fluent Bit container" + +on: + pull_request: + paths: + - "containers/fluent-bit/**" + - ".github/workflows/build-fluent-bit-container.yaml" + - "!**.md" + push: + branches: + - 'main' + paths: + - "containers/fluent-bit/**" + - ".github/workflows/build-fluent-bit-container.yaml" + - "!**.md" + +jobs: + build: + runs-on: ${{ vars.IMAGE_BUILD_RUNNER || 'ubuntu-20.04' }} + steps: + - uses: actions/checkout@v3 + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check TAG file + working-directory: containers + run: | + result="$(./tag_exists moco/fluent-bit fluent-bit)" + if [ "$result" = ok ]; then + exit 1 + fi + echo "TAG=$(cat ./fluent-bit/TAG)" >> $GITHUB_ENV + + - uses: docker/build-push-action@v4 + with: + context: containers/fluent-bit/. + platforms: linux/amd64,linux/arm64 + push: ${{ github.ref == 'refs/heads/main' }} + tags: ghcr.io/cybozu-go/moco/fluent-bit:${{ env.TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index cbdd959ee..b8c3e9f14 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -4,10 +4,14 @@ on: push: branches: [main] tags: ["v*"] - paths-ignore: ['**.md'] + paths-ignore: + - '**.md' + - "containers/**" pull_request: types: [opened, synchronize] - paths-ignore: ['**.md'] + paths-ignore: + - '**.md' + - "containers/**" env: cache-version: 1 diff --git a/containers/fluent-bit/.dockerignore b/containers/fluent-bit/.dockerignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/containers/fluent-bit/.dockerignore @@ -0,0 +1 @@ +* diff --git a/containers/fluent-bit/Dockerfile b/containers/fluent-bit/Dockerfile new file mode 100644 index 000000000..4c318c300 --- /dev/null +++ b/containers/fluent-bit/Dockerfile @@ -0,0 +1,74 @@ +# Fluent Bit container image + +# Stage1: build from source +FROM quay.io/cybozu/ubuntu-dev:22.04 AS builder + +ARG FLUENT_BIT_VERSION=2.0.9 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# https://github.com/fluent/fluent-bit/blob/master/dockerfiles/Dockerfile +ARG FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v$FLUENT_BIT_VERSION.tar.gz +ENV FLB_SOURCE $FLB_TARBALL +RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/ + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get -y install --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + cmake \ + make \ + tar \ + libssl-dev \ + libsasl2-dev \ + pkg-config \ + zlib1g-dev \ + libpq-dev \ + flex \ + bison \ + && curl -L -o "/tmp/fluent-bit.tar.gz" ${FLB_SOURCE} \ + && cd tmp/ && mkdir fluent-bit \ + && tar zxfv fluent-bit.tar.gz -C ./fluent-bit --strip-components=1 \ + && cd fluent-bit/build/ \ + && rm -rf /tmp/fluent-bit/build/* + +WORKDIR /tmp/fluent-bit/build/ +RUN cmake -DFLB_RELEASE=On \ + -DFLB_TRACE=Off \ + -DFLB_JEMALLOC=On \ + -DFLB_TLS=On \ + -DFLB_SHARED_LIB=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_HTTP_SERVER=On \ + -DFLB_IN_SYSTEMD=Off \ + -DFLB_OUT_KAFKA=On \ + -DFLB_CONFIG_YAML=No \ + .. \ + && make -j $(getconf _NPROCESSORS_ONLN) \ + && strip bin/fluent-bit \ + && mkdir -p /fluent-bit/bin \ + && cp bin/fluent-bit /fluent-bit/bin/ \ + && mkdir -p /fluent-bit/etc \ + && cd /tmp/fluent-bit \ + && cp -a conf/fluent-bit.conf \ + conf/parsers.conf \ + conf/parsers_ambassador.conf \ + conf/parsers_java.conf \ + conf/parsers_extra.conf \ + conf/parsers_openstack.conf \ + conf/parsers_cinder.conf \ + conf/plugins.conf \ + /fluent-bit/etc/ \ + && cp LICENSE /fluent-bit/ + +# Stage2: setup runtime container +FROM quay.io/cybozu/ubuntu:22.04 + +COPY --from=builder /fluent-bit /fluent-bit + +EXPOSE 2020 + +ENTRYPOINT ["/fluent-bit/bin/fluent-bit"] +CMD ["-q", "-c", "/fluent-bit/etc/fluent-bit.conf"] diff --git a/containers/fluent-bit/README.md b/containers/fluent-bit/README.md new file mode 100644 index 000000000..93c2014af --- /dev/null +++ b/containers/fluent-bit/README.md @@ -0,0 +1,11 @@ +Fluent Bit container +==================== + +Build Docker container image for [Fluent Bit][], Log Processor and Forwarder. + +Docker images +------------- + +Docker images are available on [ghcr.io](https://github.com/cybozu-go/moco/pkgs/container/moco/fluent-bit) + +[Fluent Bit]: https://fluentbit.io/ diff --git a/containers/fluent-bit/TAG b/containers/fluent-bit/TAG new file mode 100644 index 000000000..54fd9a40d --- /dev/null +++ b/containers/fluent-bit/TAG @@ -0,0 +1 @@ +2.0.9.1 diff --git a/containers/tag_exists b/containers/tag_exists new file mode 100755 index 000000000..08e76ac7a --- /dev/null +++ b/containers/tag_exists @@ -0,0 +1,36 @@ +#!/bin/sh -e + +if [ -z "$GITHUB_REPOSITORY_OWNER" ]; then + GITHUB_REPOSITORY_OWNER="cybozu-go" +fi + +if [ $# -eq 0 ]; then + echo "Usage: tag_exists NAME" + exit 1 +fi + +NAME="$1" +if [ $# -eq 2 ]; then + DIR="$2" +else + DIR="$NAME" +fi +TAG=$(cat "$DIR"/TAG) + +HTTP_STATUS=$(curl -sSL -w "%{http_code}" "https://ghcr.io/token?scope=repository%3A$GITHUB_REPOSITORY_OWNER%2F$NAME%3Apull&service=ghcr.io" -o /dev/null) +# If a non-existent image is specified, 403 will be returned. +# {"errors":[{"code":"DENIED","message":"requested access to the resource is denied"}]} +if [ "$HTTP_STATUS" = "403" ]; then + echo "ng" + exit 0 +fi + +TOKEN=$(curl -sSfL "https://ghcr.io/token?scope=repository%3A$GITHUB_REPOSITORY_OWNER%2F$NAME%3Apull&service=ghcr.io" | jq -r .token) +RESULT=$(curl -H "Authorization: Bearer $TOKEN" -sSfL "https://ghcr.io/v2/$GITHUB_REPOSITORY_OWNER/$NAME/tags/list" | jq --arg tag "$TAG" '.tags[] | select(. == $tag)') + +if [ -z "$RESULT" ]; then + echo "ng" + exit 0 +fi + +echo "ok"