Merge pull request #17 from harlem88/codegen-scripts

Codegen scripts

.gitignore

@@ -12,8 +12,11 @@ __pycache__/
 # Python virtual environments
 .venv/
 
-#Rust dist directory
+# Rust dist directory
 target/
 
+# JetBrains IDE directory
 .idea/
 
+# Codegen output directory
+output/

Makefile

@@ -0,0 +1,111 @@
+# Copyright 2023 SECO Mind Srl
+# SPDX-License-Identifier: Apache-2.0
+
+# This Makefile serves to generate the gRPC interface and
+# the related messages from .proto file.
+
+# we want bash as shell
+SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \
+         else if [ -x BASH_PATH="$(command -v bash)" ]; then echo $$BASH_PATH; \
+         else echo sh; fi; fi)
+
+# Set O variable if not already done on the command line;
+# or avoid confusing packages that can use the O=<dir> syntax for out-of-tree
+# build by preventing it from being forwarded to sub-make calls.
+ifneq ("$(origin O)", "command line")
+O := $(CURDIR)/output
+endif
+
+# Remove the trailing '/.' from $(O) as it can be added by the makefile wrapper
+# installed in the $(O) directory.
+# Also remove the trailing '/' the user can set when on the command line.
+override O := $(patsubst %/,%,$(patsubst %.,%,$(O)))
+# Make sure $(O) actually exists before calling realpath on it; this is to
+# avoid empty CANONICAL_O in case on non-existing entry.
+CANONICAL_O := $(shell mkdir -p $(O) >/dev/null 2>&1)$(realpath $(O))
+
+CANONICAL_CURDIR = $(realpath $(CURDIR))
+
+PROTO_DIR = $(CANONICAL_CURDIR)/proto
+RUST_LANG_DIR = $(CANONICAL_CURDIR)/rust
+PYTHON_LANG_DIR = $(CANONICAL_CURDIR)/python
+
+BASE_DIR := $(CANONICAL_O)
+$(if $(BASE_DIR),, $(error output directory "$(O)" does not exist))
+
+# download-location
+DL_DIR := $(shell mkdir -p $(BASE_DIR)/dl >/dev/null 2>&1)$(BASE_DIR)/dl
+
+BUILD_DIR := $(BASE_DIR)/build
+RUST_BUILD_DIR := $(BUILD_DIR)/rust
+PYTHON_BUILD_DIR := $(BUILD_DIR)/python
+
+FILES=$(wildcard proto/astarteplatform/msghub/*.proto)
+
+PROTOC_CHECK_SCRIPT=$(CANONICAL_CURDIR)/scripts/protoc_check.sh
+
+RUST_LANG=$(RUST_BUILD_DIR)/astarte-message-hub-proto
+RUST_CODEGEN_SCRIPT=$(CANONICAL_CURDIR)/scripts/rust_codegen.sh
+
+PYTHON_LANG=$(PYTHON_BUILD_DIR)/astarteplatform
+PYTHON_CODEGEN_SCRIPT=$(CANONICAL_CURDIR)/scripts/python_codegen.sh
+
+# This is our default rule, so must come first
+.PHONY: all
+all : $(RUST_LANG) $(PYTHON_LANG)
+
+$(RUST_LANG): $(FILES) $(RUST_CODEGEN_SCRIPT)
+		mkdir -p $(RUST_BUILD_DIR)
+		$(RUST_CODEGEN_SCRIPT) codegen $(PROTO_DIR) $(RUST_LANG_DIR) $(RUST_BUILD_DIR)
+
+$(PYTHON_LANG): $(FILES) $(PYTHON_CODEGEN_SCRIPT)
+		mkdir -p $(PYTHON_BUILD_DIR)
+		$(PYTHON_CODEGEN_SCRIPT) codegen $(PROTO_DIR) $(PYTHON_LANG_DIR) $(PYTHON_BUILD_DIR) $(DL_DIR)
+
+.PHONY: protoc-check
+protoc-check: $(PROTOC_CHECK_SCRIPT)
+		$(PROTOC_CHECK_SCRIPT)
+
+.PHONY: rust
+rust: protoc-check $(RUST_LANG)
+
+.PHONY: python
+python: protoc-check $(PYTHON_LANG)
+
+.PHONY: rust-install
+rust-install: $(RUST_LANG)
+		$(RUST_CODEGEN_SCRIPT) install_code $(RUST_BUILD_DIR) $(RUST_LANG_DIR)
+
+.PHONY: python-install
+python-install: $(PYTHON_LANG)
+		$(PYTHON_CODEGEN_SCRIPT) install_code $(PYTHON_BUILD_DIR) $(PYTHON_LANG_DIR)
+
+.PHONY: install
+install: rust-install python-install
+
+.PHONY: clean
+clean:
+		rm -rf $(BUILD_DIR)
+
+.PHONY: rust-dirclean
+rust-dirclean:
+		rm -rf $(RUST_BUILD_DIR)
+
+.PHONY: python-dirclean
+python-dirclean:
+		rm -rf $(PYTHON_BUILD_DIR)
+
+.PHONY: help
+help:
+		@echo 'Cleaning:'
+		@echo '  clean                  - delete all files created by build'
+		@echo
+		@echo 'Build:'
+		@echo '  all                    - Build everything and generate the code for the various languages'
+		@echo '  install                - Install files into repo folder'
+		@echo
+		@echo 'Language-specific:'
+		@echo '  <lang>                  - Build, install <lang> and all its dependencies and generate <lang> code'
+		@echo '  <lang>-install          - Install <lang> files into the repo <lang> folder'
+		@echo '  <lang>-dirclean         - Remove <lang> build directory'
+

README.md

@@ -8,3 +8,45 @@ SPDX-License-Identifier: Apache-2.0
 
 This repository contains the `.proto` files used by the Astarte message hub as well as by any
 client that wishes to connect itself to the message hub.
+
+## Code generation
+
+This project contains a Makefile that allows you to generate code for all supported languages.
+Currently, the following languages are supported:
+- Rust
+- Python.
+
+### Build 
+
+Build everything and generate the code for the various languages.
+```shell
+make
+```
+
+### Install 
+
+Install all language code files generated by the `all` rule in the repo folder.
+```shell
+make install
+```
+
+### Build language
+
+Build and generate code for specific language.
+```shell
+make rust
+```
+
+### Install the generated code
+
+Install language-code files generated by the `<lang>` rule in the repo folder.
+```shell
+make rust-install
+```
+
+### Help
+
+Run `help` for more details on the rules defined in the makefile:
+```shell
+make help
+```

python/protoc.py

@@ -55,10 +55,12 @@ def run_protoc(grpc_python_plugin: str, proto_root_fld: Path, out_fld: Path):
 
     parser = argparse.ArgumentParser()
     parser.add_argument("grpc_python_plugin")
+    parser.add_argument("--proto_dir", default=python_fld.parent.joinpath("proto"))
+    parser.add_argument("--out_dir", default=python_fld)
     args = parser.parse_args()
 
     run_protoc(
         args.grpc_python_plugin,
-        proto_root_fld=python_fld.parent.joinpath("proto"),
-        out_fld=python_fld,
+        proto_root_fld=Path(args.proto_dir),
+        out_fld=args.out_dir,
     )

scripts/protoc_check.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 SECO Mind Srl
+# SPDX-License-Identifier: Apache-2.0
+
+set -eEuo pipefail
+
+PROTOC_VERSION='25.4'
+version=$(protoc --version | cut -d ' ' -f 2)
+
+if [[ $version != "$PROTOC_VERSION" ]]; then
+    echo "incompatible protoc version $version, expected $PROTOC_VERSION" >&2
+    exit 1
+fi

scripts/python_codegen.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 SECO Mind Srl
+# SPDX-License-Identifier: Apache-2.0
+
+set -eEuo pipefail
+
+codegen () {
+    PROTO_DIR=$1
+    PROJECT_DIR=$2
+    OUT_DIR=$3
+    DL_DIR=$4
+
+    # Remove old code
+    if [ -d "$OUT_DIR/astarteplatform" ]; then
+        rm -r "$OUT_DIR/astarteplatform"
+    fi
+
+    if [ ! -d "$DL_DIR/grpc" ]; then
+        cd "$DL_DIR"
+        git clone -b v1.58.1 https://github.com/grpc/grpc
+        cd grpc
+        git submodule update --init
+    fi
+
+    if [ ! -f "$DL_DIR/grpc/grpc_python_plugin" ]; then
+        cd "$DL_DIR/grpc"
+        cmake .
+        make -j$(nproc) grpc_python_plugin
+        python3 -m pip install --upgrade pip
+        python3 -m pip install termcolor
+        cd -
+    fi
+
+    python3 "$PROJECT_DIR/protoc.py" "$DL_DIR/grpc/grpc_python_plugin" --proto_dir "$PROTO_DIR" --out_dir "$OUT_DIR"
+    }
+
+install_code (){
+      OUT_DIR=$1
+      INSTALL_DIR=$2
+
+      install -d "$OUT_DIR"/astarteplatform "$INSTALL_DIR"/astarteplatform
+}
+
+if [ "$1" = "codegen" ]; then
+  codegen "$2" "$3" "$4" "$5"
+elif [ "$1" = "install_code" ]; then
+    install_code "$2" "$3"
+fi

scripts/rust_codegen.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 SECO Mind Srl
+# SPDX-License-Identifier: Apache-2.0
+
+set -eEuo pipefail
+
+codegen () {
+    PROTO_DIR=$1
+    PROJECT_DIR=$2
+    OUT_DIR=$3
+
+    # Remove old code
+    if [ -f "$OUT_DIR/astarte-message-hub-proto/src/astarteplatform.msghub.rs" ]; then
+        rm "$OUT_DIR/astarte-message-hub-proto/src/astarteplatform.msghub.rs"
+    fi
+
+    cargo run --manifest-path "$PROJECT_DIR"/Cargo.toml -p rust-codegen -- -p "$PROTO_DIR" -o "$OUT_DIR"
+}
+
+install_code (){
+  OUT_DIR=$1
+  INSTALL_DIR=$2
+
+  install "$OUT_DIR"/astarteplatform.msghub.rs "$INSTALL_DIR"/astarte-message-hub-proto/src/astarteplatform.msghub.rs
+}
+
+if [ "$1" = "codegen" ]; then
+  codegen "$2" "$3" "$4"
+elif [ "$1" = "install_code" ]; then
+    install_code "$2" "$3"
+fi