diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7e2cc72e5b..ea28d9ddcb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -48,19 +48,6 @@ jobs: run: | echo /home/runner/go/bin >> $GITHUB_PATH echo /usr/local/bin >> $GITHUB_PATH - - name: Install protoc - run: | - set -eux -o pipefail - PROTOC_VERSION=27.1 - PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP - sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - sudo chmod +x /usr/local/bin/protoc - sudo find /usr/local/include -type f | xargs sudo chmod a+r - sudo find /usr/local/include -type d | xargs sudo chmod a+rx - rm -f $PROTOC_ZIP - ls /usr/local/include/google/protobuf/ - name: Install pandoc run: | set -eux -o pipefail diff --git a/docs/user-guide/sources/user-defined-sources.md b/docs/user-guide/sources/user-defined-sources.md index 71d8f1dc5d..2665973a3f 100644 --- a/docs/user-guide/sources/user-defined-sources.md +++ b/docs/user-guide/sources/user-defined-sources.md @@ -12,7 +12,7 @@ Check the links below to see the examples for different languages. - [Golang](https://github.com/numaproj/numaflow-go/tree/main/pkg/sourcer/examples/simple_source/) - [Java](https://github.com/numaproj/numaflow-java/tree/main/examples/src/main/java/io/numaproj/numaflow/examples/source/simple/) -- [Python](https://github.com/numaproj/numaflow-python/tree/main/examples/source/simple-source) +- [Python](https://github.com/numaproj/numaflow-python/tree/main/examples/source/simple_source) After building a docker image for the written user-defined source, specify the image as below in the vertex spec. diff --git a/hack/generate-proto.sh b/hack/generate-proto.sh index f2501d36fa..614ded3e8a 100755 --- a/hack/generate-proto.sh +++ b/hack/generate-proto.sh @@ -7,52 +7,51 @@ set -o pipefail source $(dirname $0)/library.sh header "generating proto files" -ensure_protobuf ensure_vendor -if [ "`command -v protoc-gen-gogo`" = "" ]; then - go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogo -fi - -if [ "`command -v protoc-gen-gogofast`" = "" ]; then - go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast -fi - -if [ "`command -v gogoproto`" = "" ]; then - go install -mod=vendor ./vendor/github.com/gogo/protobuf/gogoproto -fi - -if [ "`command -v protoc-gen-go`" = "" ]; then - go install -mod=vendor ./vendor/google.golang.org/protobuf/cmd/protoc-gen-go -fi - -if [ "`command -v protoc-gen-go-grpc`" = "" ]; then - go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc -fi - -if [ "`command -v protoc-gen-grpc-gateway`" = "" ]; then - go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway -fi - -if [ "`command -v protoc-gen-openapiv2`" = "" ]; then - go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 -fi - -if [ "`command -v goimports`" = "" ]; then - go install -mod=vendor ./vendor/golang.org/x/tools/cmd/goimports -fi - -export PATH="$(go env GOPATH)/bin:${PATH}" - make_fake_paths export GOPATH="${FAKE_GOPATH}" +export PATH="${GOPATH}/bin:${PATH}" cd "${FAKE_REPOPATH}" +install-protobuf() { + # protobuf version + PROTOBUF_VERSION=27.2 + PB_REL="https://github.com/protocolbuffers/protobuf/releases" + OS=$(uname_os) + ARCH=$(uname_arch) + + echo "OS: $OS ARCH: $ARCH" + BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip + if [[ "$OS" = "darwin" ]]; then + BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-osx-universal_binary.zip + elif [[ "$OS" = "linux" ]]; then + BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip + fi + echo "Downloading $BINARY_URL" + + tmp=$(mktemp -d) + trap 'rm -rf ${tmp}' EXIT + + curl -sL -o ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip $BINARY_URL + unzip ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip -d ${GOPATH} +} + +install-protobuf + +go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogo +go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast +go install -mod=vendor ./vendor/github.com/gogo/protobuf/gogoproto +go install -mod=vendor ./vendor/google.golang.org/protobuf/cmd/protoc-gen-go +go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc +go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway +go install -mod=vendor ./vendor/github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 +go install -mod=vendor ./vendor/golang.org/x/tools/cmd/goimports go install -mod=vendor ./vendor/k8s.io/code-generator/cmd/go-to-protobuf export GO111MODULE="off" -${GOPATH}/bin/go-to-protobuf \ +go-to-protobuf \ --go-header-file=./hack/boilerplate/boilerplate.go.txt \ --packages=github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1 \ --apimachinery-packages=+k8s.io/apimachinery/pkg/util/intstr,+k8s.io/apimachinery/pkg/api/resource,k8s.io/apimachinery/pkg/runtime/schema,+k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1,k8s.io/api/policy/v1beta1 \ diff --git a/hack/library.sh b/hack/library.sh index e76e4cee43..45928775b4 100644 --- a/hack/library.sh +++ b/hack/library.sh @@ -51,24 +51,45 @@ ensure_pandoc() { fi } -# TODO: Install it in Makefile -ensure_protobuf() { - if [ "`command -v protoc`" = "" ]; then - warning "Please install protobuf with - brew install protobuf" - exit 1 - fi -} - ensure_node(){ if [ "`command -v node`" = "" ]; then warning "Please install node with - brew install node" exit 1 - fi + fi } ensure_yarn(){ if [ "`command -v yarn`" = "" ]; then warning "Please install yarn with - brew install yarn" exit 1 - fi + fi +} + +uname_os() { + os=$(uname -s | tr '[:upper:]' '[:lower:]') + case "$os" in + msys*) os="windows" ;; + mingw*) os="windows" ;; + cygwin*) os="windows" ;; + win*) os="windows" ;; + sunos) [ "$(uname -o)" = "illumos" ] && os=illumos ;; + esac + echo "$os" +} + +uname_arch() { + arch=$(uname -m) + case $arch in + x86_64) arch="amd64" ;; + x86) arch="386" ;; + i686) arch="386" ;; + i386) arch="386" ;; + i86pc) arch="amd64" ;; + aarch64) arch="arm64" ;; + armv5*) arch="armv5" ;; + armv6*) arch="armv6" ;; + armv7*) arch="armv7" ;; + loongarch64) arch="loong64" ;; + esac + echo "${arch}" } diff --git a/pkg/apis/proto/daemon/daemon.pb.go b/pkg/apis/proto/daemon/daemon.pb.go index c895795df3..6cb88c9f53 100644 --- a/pkg/apis/proto/daemon/daemon.pb.go +++ b/pkg/apis/proto/daemon/daemon.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v5.27.1 +// protoc v5.27.2 // source: pkg/apis/proto/daemon/daemon.proto package daemon diff --git a/pkg/apis/proto/daemon/daemon_grpc.pb.go b/pkg/apis/proto/daemon/daemon_grpc.pb.go index 6224351683..61e15a2a62 100644 --- a/pkg/apis/proto/daemon/daemon_grpc.pb.go +++ b/pkg/apis/proto/daemon/daemon_grpc.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v5.27.1 +// - protoc v5.27.2 // source: pkg/apis/proto/daemon/daemon.proto package daemon diff --git a/pkg/apis/proto/isb/message.pb.go b/pkg/apis/proto/isb/message.pb.go index 9dbc790e27..03f6308751 100644 --- a/pkg/apis/proto/isb/message.pb.go +++ b/pkg/apis/proto/isb/message.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v5.27.1 +// protoc v5.27.2 // source: pkg/apis/proto/isb/message.proto package isb diff --git a/pkg/apis/proto/wmb/wmb.pb.go b/pkg/apis/proto/wmb/wmb.pb.go index 036f86f0c5..a11dc1c385 100644 --- a/pkg/apis/proto/wmb/wmb.pb.go +++ b/pkg/apis/proto/wmb/wmb.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v5.27.1 +// protoc v5.27.2 // source: pkg/apis/proto/wmb/wmb.proto package isb