Build fluent-bit container image. (#528)

* Build fluent-bit container image.

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Remove BRANCH file

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Fix tag script.

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Use larger_runner_16core

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Fix container link

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Add debug value

Signed-off-by: d-kuro <kurosawa7620@gmail.com>

* Fix review feedback issues.

Signed-off-by: d-kuro <kurosawa7620@gmail.com>
Co-authored-by: masa213f <masa213f@users.noreply.github.com>

---------

Signed-off-by: d-kuro <kurosawa7620@gmail.com>
Co-authored-by: masa213f <masa213f@users.noreply.github.com>

.github/workflows/build-fluent-bit-container.yaml

@@ -0,0 +1,47 @@
+name: "Build Fluent Bit container"
+
+on:
+  pull_request:
+    paths:
+      - "containers/fluent-bit/**"
+      - ".github/workflows/build-fluent-bit-container.yaml"
+      - "!**.md"
+  push:
+    branches:
+      - 'main'
+    paths:
+      - "containers/fluent-bit/**"
+      - ".github/workflows/build-fluent-bit-container.yaml"
+      - "!**.md"
+
+jobs:
+  build:
+    runs-on: ${{ vars.IMAGE_BUILD_RUNNER || 'ubuntu-20.04' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check TAG file
+        working-directory: containers
+        run: |
+          result="$(./tag_exists moco/fluent-bit fluent-bit)"
+          if [ "$result" = ok ]; then
+            exit 1
+          fi
+          echo "TAG=$(cat ./fluent-bit/TAG)" >> $GITHUB_ENV
+
+      - uses: docker/build-push-action@v4
+        with:
+          context: containers/fluent-bit/.
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.ref == 'refs/heads/main' }}
+          tags: ghcr.io/cybozu-go/moco/fluent-bit:${{ env.TAG }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/ci-e2e.yaml

@@ -4,10 +4,14 @@ on:
   push:
     branches: [main]
     tags: ["v*"]
-    paths-ignore: ['**.md']
+    paths-ignore:
+      - '**.md'
+      - "containers/**"
   pull_request:
     types: [opened, synchronize]
-    paths-ignore: ['**.md']
+    paths-ignore:
+      - '**.md'
+      - "containers/**"
 
 env:
   cache-version: 1

containers/fluent-bit/.dockerignore

@@ -0,0 +1 @@
+*

containers/fluent-bit/Dockerfile

@@ -0,0 +1,74 @@
+# Fluent Bit container image
+
+# Stage1: build from source
+FROM quay.io/cybozu/ubuntu-dev:22.04 AS builder
+
+ARG FLUENT_BIT_VERSION=2.0.9
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# https://github.com/fluent/fluent-bit/blob/master/dockerfiles/Dockerfile
+ARG FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v$FLUENT_BIT_VERSION.tar.gz
+ENV FLB_SOURCE $FLB_TARBALL
+RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get -y install --no-install-recommends \
+    build-essential \
+    curl \
+    ca-certificates \
+    cmake \
+    make \
+    tar \
+    libssl-dev \
+    libsasl2-dev \
+    pkg-config \
+    zlib1g-dev \
+    libpq-dev \
+    flex \
+    bison \
+    && curl -L -o "/tmp/fluent-bit.tar.gz" ${FLB_SOURCE} \
+    && cd tmp/ && mkdir fluent-bit \
+    && tar zxfv fluent-bit.tar.gz -C ./fluent-bit --strip-components=1 \
+    && cd fluent-bit/build/ \
+    && rm -rf /tmp/fluent-bit/build/*
+
+WORKDIR /tmp/fluent-bit/build/
+RUN cmake -DFLB_RELEASE=On \
+          -DFLB_TRACE=Off \
+          -DFLB_JEMALLOC=On \
+          -DFLB_TLS=On \
+          -DFLB_SHARED_LIB=Off \
+          -DFLB_EXAMPLES=Off \
+          -DFLB_HTTP_SERVER=On \
+          -DFLB_IN_SYSTEMD=Off \
+          -DFLB_OUT_KAFKA=On \
+          -DFLB_CONFIG_YAML=No \
+          .. \
+    && make -j $(getconf _NPROCESSORS_ONLN) \
+    && strip bin/fluent-bit \
+    && mkdir -p /fluent-bit/bin \
+    && cp bin/fluent-bit /fluent-bit/bin/ \
+    && mkdir -p /fluent-bit/etc \
+    && cd /tmp/fluent-bit \
+    && cp -a conf/fluent-bit.conf \
+        conf/parsers.conf \
+        conf/parsers_ambassador.conf \
+        conf/parsers_java.conf \
+        conf/parsers_extra.conf \
+        conf/parsers_openstack.conf \
+        conf/parsers_cinder.conf \
+        conf/plugins.conf \
+        /fluent-bit/etc/ \
+    && cp LICENSE /fluent-bit/
+
+# Stage2: setup runtime container
+FROM quay.io/cybozu/ubuntu:22.04
+
+COPY --from=builder /fluent-bit /fluent-bit
+
+EXPOSE 2020
+
+ENTRYPOINT ["/fluent-bit/bin/fluent-bit"]
+CMD ["-q", "-c", "/fluent-bit/etc/fluent-bit.conf"]

containers/fluent-bit/README.md

@@ -0,0 +1,11 @@
+Fluent Bit container
+====================
+
+Build Docker container image for [Fluent Bit][], Log Processor and Forwarder.
+
+Docker images
+-------------
+
+Docker images are available on [ghcr.io](https://github.com/cybozu-go/moco/pkgs/container/moco/fluent-bit)
+
+[Fluent Bit]: https://fluentbit.io/

containers/fluent-bit/TAG

@@ -0,0 +1 @@
+2.0.9.1

containers/tag_exists

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+
+if [ -z "$GITHUB_REPOSITORY_OWNER" ]; then
+  GITHUB_REPOSITORY_OWNER="cybozu-go"
+fi
+
+if [ $# -eq 0 ]; then
+    echo "Usage: tag_exists NAME"
+    exit 1
+fi
+
+NAME="$1"
+if [ $# -eq 2 ]; then
+    DIR="$2"
+else
+    DIR="$NAME"
+fi
+TAG=$(cat "$DIR"/TAG)
+
+HTTP_STATUS=$(curl -sSL -w "%{http_code}" "https://ghcr.io/token?scope=repository%3A$GITHUB_REPOSITORY_OWNER%2F$NAME%3Apull&service=ghcr.io" -o /dev/null)
+# If a non-existent image is specified, 403 will be returned.
+# {"errors":[{"code":"DENIED","message":"requested access to the resource is denied"}]}
+if [ "$HTTP_STATUS" = "403" ]; then
+    echo "ng"
+    exit 0
+fi
+
+TOKEN=$(curl -sSfL "https://ghcr.io/token?scope=repository%3A$GITHUB_REPOSITORY_OWNER%2F$NAME%3Apull&service=ghcr.io" | jq -r .token)
+RESULT=$(curl -H "Authorization: Bearer $TOKEN" -sSfL "https://ghcr.io/v2/$GITHUB_REPOSITORY_OWNER/$NAME/tags/list" | jq --arg tag "$TAG" '.tags[] | select(. == $tag)')
+
+if [ -z "$RESULT" ]; then
+    echo "ng"
+    exit 0
+fi
+
+echo "ok"