refactor(core): unify core, vdp and model repos

.env

@@ -1,18 +1,15 @@
 # docker compose project name
-COMPOSE_PROJECT_NAME=instill-vdp
-
-# build from scratch or not at launch, which will build all sources from scrach. Default to false.
-BUILD=false
+COMPOSE_PROJECT_NAME=instill-core
 
 # docker compose profiles to selectively launch components for developing the latest codebase of the specified component.
 # the value can be all, exclude-api-gateway, exclude-mgmt, exclude-pipeline, exclude-model, exclude-controller-model, or exclude-console.
 PROFILE=all
 
-# system-wise config path (all core, vdp, and model projects must use the same path)
-SYSTEM_CONFIG_PATH=~/.config/instill
+# build from scratch or not at launch, which will build all sources from scrach. Default to false.
+BUILD=false
 
-# configuration directory path for docker build
-BUILD_CONFIG_DIR_PATH=.
+# system-wise config path (all vdp, model and artifact projects must use the same path)
+SYSTEM_CONFIG_PATH=~/.config/instill
 
 # extra parameters for helm integration test running in docker
 DOCKER_HELM_IT_EXTRA_PARAMS=
@@ -23,29 +20,49 @@ USAGE_ENABLED=true
 # flag to enable observability stack
 OBSERVE_ENABLED=false
 
-# max data size in MB which pipeline and model backend accept to process
-MAX_DATA_SIZE=12
+# This flag is used for integration test in which dummy model is used instead of pulling model from GitHub, HuggingFace or ArtiVC.
+# The reason is reducing the impact of network trouble during integration test
+# The default value is alway false, only set when running `make integration-test`
+ITMODE_ENABLED=false
+
+# flag to enable model-backend creating predploy models
+INITMODEL_ENABLED=false
+
+# configuration directory path for observability stack
+OBSERVE_CONFIG_DIR_PATH=./configs
+
+# Instill Core instance host
+INSTILL_CORE_HOST=localhost
 
 # container build
 DOCKER_BUILDKIT=1
 COMPOSE_DOCKER_CLI_BUILD=1
 
+# max data size in MB which pipeline-/model-/artifact-backend accept to process
+MAX_DATA_SIZE=12
+
 # version
 ALPINE_VERSION=3.16
 
 GOLANG_VERSION=1.21
-KRAKEND_CE_VERSION=2.1.3
+KRAKEND_CE_VERSION=2.5.0
 ARTIVC_VERSION=0.10.0
 K6_VERSION=0.44.0
 
-# Instill Core
-INSTILL_CORE_VERSION=0.12.0-beta
-INSTILL_CORE_HOST=localhost
-
 # api-gateway
+API_GATEWAY_IMAGE=instill/api-gateway
+API_GATEWAY_VERSION=0.10.0-beta
 API_GATEWAY_HOST=api-gateway
 API_GATEWAY_PORT=8080
-API_GATEWAY_VERSION=0.10.0-beta
+API_GATEWAY_STATS_PORT=8070
+API_GATEWAY_METRICS_PORT=8071
+
+# mgmt-backend
+MGMT_BACKEND_IMAGE=instill/mgmt-backend
+MGMT_BACKEND_VERSION=0.11.0-beta
+MGMT_BACKEND_HOST=mgmt-backend
+MGMT_BACKEND_PRIVATEPORT=3084
+MGMT_BACKEND_PUBLICPORT=8084
 
 # pipeline-backend
 PIPELINE_BACKEND_IMAGE=instill/pipeline-backend
@@ -54,39 +71,124 @@ PIPELINE_BACKEND_HOST=pipeline-backend
 PIPELINE_BACKEND_PRIVATEPORT=3081
 PIPELINE_BACKEND_PUBLICPORT=8081
 
-# mgmt-backend
-MGMT_BACKEND_HOST=mgmt-backend
-MGMT_BACKEND_PRIVATEPORT=3084
-MGMT_BACKEND_PUBLICPORT=8084
-MGMT_BACKEND_VERSION=0.11.0-beta
-
 # model-backend
+MODEL_BACKEND_IMAGE=instill/model-backend
+MODEL_BACKEND_VERSION=0.22.0-alpha
 MODEL_BACKEND_HOST=model-backend
+MODEL_BACKEND_PRIVATEPORT=3083
 MODEL_BACKEND_PUBLICPORT=8083
 
-# socat
-SOCAT_IMAGE=alpine/socat
-SOCAT_VERSION=1.7.4.4-r0
-SOCAT_HOST=socat
-SOCAT_PORT=2375
+# controller-model
+CONTROLLER_MODEL_IMAGE=instill/controller-model
+CONTROLLER_MODEL_VERSION=0.4.0-alpha
+CONTROLLER_MODEL_HOST=controller-model
+CONTROLLER_MODEL_PRIVATEPORT=3086
+
+# console
+CONSOLE_IMAGE=instill/console
+CONSOLE_VERSION=0.39.0-beta
+CONSOLE_HOST=console
+CONSOLE_PORT=3000
+
+# ray-server
+RAY_SERVER_IMAGE=instill/ray
+RAY_SERVER_VERSION=0.2.5-alpha
+RAY_SERVER_HOST=ray-server
+RAY_SERVER_CLIENT_PORT=10001
+RAY_SERVER_DASHBOARD_PORT=8265
+RAY_SERVER_GCS_PORT=6379
+RAY_SERVER_SERVE_PORT=8000
+RAY_SERVER_SERVE_GRPC_PORT=9000
+RAY_SERVER_PROMETHEUS_PORT=8079
+RAY_SERVER_VRAM=
 
 # PostgreSQL
+POSTGRESQL_IMAGE=postgres
+POSTGRESQL_VERSION=14.1
 POSTGRESQL_HOST=pg-sql
 POSTGRESQL_PORT=5432
 
+# Elasticseach
+ELASTICSEARCH_IMAGE=elasticsearch
+ELASTICSEARCH_VERSION=7.16.2
+ELASTICSEARCH_HOST=elasticsearch
+ELASTICSEARCH_PORT=9200
+
+# Temopral
+TEMPORAL_IMAGE=temporalio/auto-setup
+TEMPORAL_VERSION=1.22.3
+TEMPORAL_HOST=temporal
+TEMPORAL_PORT=7233
+
+TEMPORAL_UI_IMAGE=temporalio/ui
+TEMPORAL_UI_VERSION=2.21.3
+TEMPORAL_UI_HOST=temporal-ui
+TEMPORAL_UI_PORT=8088
+
 # Redis
+REDIS_IMAGE=redis
+REDIS_VERSION=7.0
 REDIS_HOST=redis
 REDIS_PORT=6379
 
+# Redoc OpenAPI
+REDOC_IMAGE=redocly/redoc
+REDOC_OPENAPI_VERSION=v2.0.0-rc.70
+REDOC_OPENAPI_HOST=redoc-openapi
+REDOC_OPENAPI_PORT=3001
+
 # etcd
+ETCD_IMAGE=quay.io/coreos/etcd
+ETCD_VERSION=3.5.8
 ETCD_HOST=etcd
 ETCD_CLIENT_PORT=3379
+ETCD_PEER_PORT=3380
+ETCD_CLUSTER_TOKEN=instill-core-etcd
+ETCD_CLUSTER_STATE=new
+ETCDCTL_API=3
+ETCD_AUTO_COMPACTION_MODE=revision
+ETCD_AUTO_COMPACTION_RETENTION=1
+
+# influxdb
+INFLUXDB_IMAGE=influxdb
+INFLUXDB_VERSION=2.7
+INFLUXDB_HOST=influxdb
+INFLUXDB_PORT=8086
+
+# opengfa
+OPENFGA_IMAGE=openfga/openfga
+OPENFGA_VERSION=v1.3.7
+OPENFGA_HOST=openfga
 
 # otel
+OTEL_COLLECTOR_IMAGE=otel/opentelemetry-collector-contrib
+OTEL_COLLECTOR_VERSION=0.78.0
+OTEL_COLLECTOR_HOST=otel-collector
 OTEL_COLLECTOR_PORT=8095
+OTEL_COLLECTOR_PROMETHEUS_PORT=9001
+
+# jaeger
+JAEGER_IMAGE=jaegertracing/all-in-one
+JAEGER_VERSION=1.45.0
+JAEGER_HOST=jaeger
+JAEGER_PORT=16686
+JAEGER_LISTEN_PROTO_PORT=14250  # accept model.proto
+JAEGER_LISTEN_THRIFT_PORT=14268 # accept jaeger.thrift directly from clients
+
+# prometheus
+PROMETHEUS_IMAGE=prom/prometheus
+PROMETHEUS_VERSION=2.44.0
+PROMETHEUS_HOST=prometheus
+PROMETHEUS_PORT=9090
+
+# grafana
+GRAFANA_IMAGE=grafana/grafana
+GRAFANA_VERSION=9.5.2
+GRAFANA_HOST=grafana
+GRAFANA_PORT=3002
 
-# Temopral
-TEMPORAL_ADMIN_TOOLS_IMAGE=temporalio/admin-tools
-TEMPORAL_ADMIN_TOOLS_VERSION=1.22.3
-TEMPORAL_HOST=temporal
-TEMPORAL_PORT=7233
+# socat
+SOCAT_IMAGE=alpine/socat
+SOCAT_VERSION=1.7.4.4-r0
+SOCAT_HOST=socat
+SOCAT_PORT=2375

.github/CONTRIBUTING.md

@@ -46,7 +46,7 @@ $ make down
 
 ### Build the local images
 
-We use Docker multi-stage builds to build a `instill/vdp-compose:{latest,release}` image which will be based on to run dind (docker-in-docker) to build all the images of `Instill VDP` defined in the compose file [docker-compose.build.yml](../docker-compose.build.yml).
+We use Docker multi-stage builds to build a `instill/vdp-compose:{latest,release}` image which will be based on to run dind (docker-in-docker) to build all the images of `Instill VDP` defined in the compose file [docker-compose-build.yml](../docker-compose-build.yml).
 
 You can build the images by simply running: