chore: install protobuf within codegen (numaproj#1814)

Signed-off-by: Derek Wang <whynowy@gmail.com>

.github/workflows/ci.yaml

@@ -48,19 +48,6 @@ jobs:
         run: |
           echo /home/runner/go/bin >> $GITHUB_PATH
           echo /usr/local/bin >> $GITHUB_PATH
-      - name: Install protoc
-        run: |
-          set -eux -o pipefail
-          PROTOC_VERSION=27.1
-          PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip
-          curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP
-          sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
-          sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*'
-          sudo chmod +x /usr/local/bin/protoc
-          sudo find /usr/local/include -type f | xargs sudo chmod a+r
-          sudo find /usr/local/include -type d | xargs sudo chmod a+rx
-          rm -f $PROTOC_ZIP
-          ls /usr/local/include/google/protobuf/
       - name: Install pandoc
         run: |
           set -eux -o pipefail

docs/user-guide/sources/user-defined-sources.md

@@ -12,7 +12,7 @@ Check the links below to see the examples for different languages.
 
 - [Golang](https://github.com/numaproj/numaflow-go/tree/main/pkg/sourcer/examples/simple_source/)
 - [Java](https://github.com/numaproj/numaflow-java/tree/main/examples/src/main/java/io/numaproj/numaflow/examples/source/simple/)
-- [Python](https://github.com/numaproj/numaflow-python/tree/main/examples/source/simple-source)
+- [Python](https://github.com/numaproj/numaflow-python/tree/main/examples/source/simple_source)
 
 After building a docker image for the written user-defined source, specify the image as below in the vertex spec.
 

hack/generate-proto.sh

@@ -7,52 +7,51 @@ set -o pipefail
 source $(dirname $0)/library.sh
 header "generating proto files"
 
-ensure_protobuf
 ensure_vendor
 
-if [ "`command -v protoc-gen-gogo`" = "" ]; then
-  go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogo
-fi
-
-if [ "`command -v protoc-gen-gogofast`" = "" ]; then
-  go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast
-fi
-
-if [ "`command -v gogoproto`" = "" ]; then
-  go install -mod=vendor ./vendor/github.com/gogo/protobuf/gogoproto
-fi
-
-if [ "`command -v protoc-gen-go`" = "" ]; then
-  go install -mod=vendor ./vendor/google.golang.org/protobuf/cmd/protoc-gen-go
-fi
-
-if [ "`command -v protoc-gen-go-grpc`" = "" ]; then
-  go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc
-fi
-
-if [ "`command -v protoc-gen-grpc-gateway`" = "" ]; then
-  go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
-fi
-
-if [ "`command -v protoc-gen-openapiv2`" = "" ]; then
-  go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
-fi
-
-if [ "`command -v goimports`" = "" ]; then
-  go install -mod=vendor ./vendor/golang.org/x/tools/cmd/goimports
-fi
-
-export PATH="$(go env GOPATH)/bin:${PATH}"
-
 make_fake_paths
 export GOPATH="${FAKE_GOPATH}"
+export PATH="${GOPATH}/bin:${PATH}"
 cd "${FAKE_REPOPATH}"
 
+install-protobuf() {
+  # protobuf version
+  PROTOBUF_VERSION=27.2
+  PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+  OS=$(uname_os)
+  ARCH=$(uname_arch)
+
+  echo "OS: $OS  ARCH: $ARCH"
+  BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip
+  if [[ "$OS" = "darwin" ]]; then
+    BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-osx-universal_binary.zip
+  elif [[ "$OS" = "linux" ]]; then
+    BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
+  fi
+  echo "Downloading $BINARY_URL"
+
+  tmp=$(mktemp -d)
+  trap 'rm -rf ${tmp}' EXIT
+
+  curl -sL -o ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip $BINARY_URL
+  unzip ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip -d ${GOPATH}
+}
+
+install-protobuf
+
+go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogo
+go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast
+go install -mod=vendor ./vendor/github.com/gogo/protobuf/gogoproto
+go install -mod=vendor ./vendor/google.golang.org/protobuf/cmd/protoc-gen-go
+go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc
+go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
+go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
+go install -mod=vendor ./vendor/golang.org/x/tools/cmd/goimports
 go install -mod=vendor ./vendor/k8s.io/code-generator/cmd/go-to-protobuf
 
 export GO111MODULE="off"
 
-${GOPATH}/bin/go-to-protobuf \
+go-to-protobuf \
         --go-header-file=./hack/boilerplate/boilerplate.go.txt \
         --packages=github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1 \
         --apimachinery-packages=+k8s.io/apimachinery/pkg/util/intstr,+k8s.io/apimachinery/pkg/api/resource,k8s.io/apimachinery/pkg/runtime/schema,+k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1,k8s.io/api/policy/v1beta1 \

hack/library.sh

@@ -51,24 +51,45 @@ ensure_pandoc() {
   fi
 }
 
-# TODO: Install it in Makefile
-ensure_protobuf() {
-  if [ "`command -v protoc`" = "" ]; then
-    warning "Please install protobuf with - brew install protobuf"
-    exit 1
-  fi
-}
-
 ensure_node(){
   if [ "`command -v node`" = "" ]; then
       warning "Please install node with - brew install node"
       exit 1
-    fi
+  fi
 }
 
 ensure_yarn(){
   if [ "`command -v yarn`" = "" ]; then
       warning "Please install yarn with - brew install yarn"
       exit 1
-    fi
+  fi
+}
+
+uname_os() {
+  os=$(uname -s | tr '[:upper:]' '[:lower:]')
+  case "$os" in
+    msys*) os="windows" ;;
+    mingw*) os="windows" ;;
+    cygwin*) os="windows" ;;
+    win*) os="windows" ;;
+    sunos) [ "$(uname -o)" = "illumos" ] && os=illumos ;;
+  esac
+  echo "$os"
+}
+
+uname_arch() {
+  arch=$(uname -m)
+  case $arch in
+    x86_64) arch="amd64" ;;
+    x86) arch="386" ;;
+    i686) arch="386" ;;
+    i386) arch="386" ;;
+    i86pc) arch="amd64" ;;
+    aarch64) arch="arm64" ;;
+    armv5*) arch="armv5" ;;
+    armv6*) arch="armv6" ;;
+    armv7*) arch="armv7" ;;
+    loongarch64) arch="loong64" ;;
+  esac
+  echo "${arch}"
 }