From 1e2a887309a08df0f5d8fd975d8d4de36e35a956 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 7 Feb 2024 20:54:08 +0100 Subject: [PATCH 1/4] CI: Reschedule ml-automl job to run at 6 o'clock, three hours later The nightly job often randomly fails at night time, but always succeeds at day time when triggered manually. Let's check if this improves the situation. --- .github/workflows/ml-automl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ml-automl.yml b/.github/workflows/ml-automl.yml index cf0ba835..b3e8552b 100644 --- a/.github/workflows/ml-automl.yml +++ b/.github/workflows/ml-automl.yml @@ -19,7 +19,7 @@ on: # Run job each night after CrateDB nightly has been published. schedule: - - cron: '0 3 * * *' + - cron: '0 6 * * *' # Cancel in-progress jobs when pushing to the same branch. concurrency: From 97284635795ce0259c45e1748e77fd35f17ba90c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 7 Feb 2024 20:58:12 +0100 Subject: [PATCH 2/4] CI: Reconfigure CrateDB container to use `CRATE_HEAP_SIZE: 4g` Recently, we observed problems on CI when working with larger datasets. There is no reason to run CrateDB on 512 MB, because GitHub runners provide 16 GB of RAM. --- .github/workflows/framework-apache-superset.yml | 2 ++ .github/workflows/lang-java-jooq.yml | 2 ++ .github/workflows/lang-java-maven.yml | 2 ++ .github/workflows/lang-npgsql.yml | 2 ++ .github/workflows/lang-php-amphp.yml | 2 ++ .github/workflows/lang-php-pdo.yml | 2 ++ .github/workflows/lang-ruby.yml | 2 ++ .github/workflows/ml-automl.yml | 2 ++ .github/workflows/ml-langchain.yml | 2 ++ .github/workflows/ml-mlflow.yml | 2 ++ 10 files changed, 20 insertions(+) diff --git a/.github/workflows/framework-apache-superset.yml b/.github/workflows/framework-apache-superset.yml index bb1ce098..5217c008 100644 --- a/.github/workflows/framework-apache-superset.yml +++ b/.github/workflows/framework-apache-superset.yml @@ -44,6 +44,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g name: Superset ${{ matrix.superset-version }}, Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/lang-java-jooq.yml b/.github/workflows/lang-java-jooq.yml index 7285a014..a312ffad 100644 --- a/.github/workflows/lang-java-jooq.yml +++ b/.github/workflows/lang-java-jooq.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/lang-java-maven.yml b/.github/workflows/lang-java-maven.yml index 4bfd88b6..3147b526 100644 --- a/.github/workflows/lang-java-maven.yml +++ b/.github/workflows/lang-java-maven.yml @@ -48,6 +48,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/lang-npgsql.yml b/.github/workflows/lang-npgsql.yml index e0a7f0d0..2178d1c3 100644 --- a/.github/workflows/lang-npgsql.yml +++ b/.github/workflows/lang-npgsql.yml @@ -53,6 +53,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/lang-php-amphp.yml b/.github/workflows/lang-php-amphp.yml index 31e04140..811e9bc4 100644 --- a/.github/workflows/lang-php-amphp.yml +++ b/.github/workflows/lang-php-amphp.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/lang-php-pdo.yml b/.github/workflows/lang-php-pdo.yml index cabdbc96..621080a3 100644 --- a/.github/workflows/lang-php-pdo.yml +++ b/.github/workflows/lang-php-pdo.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/lang-ruby.yml b/.github/workflows/lang-ruby.yml index d224c64a..0ebf6c5e 100644 --- a/.github/workflows/lang-ruby.yml +++ b/.github/workflows/lang-ruby.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: diff --git a/.github/workflows/ml-automl.yml b/.github/workflows/ml-automl.yml index b3e8552b..b220e71e 100644 --- a/.github/workflows/ml-automl.yml +++ b/.github/workflows/ml-automl.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/.github/workflows/ml-langchain.yml b/.github/workflows/ml-langchain.yml index 671502ad..46d55b55 100644 --- a/.github/workflows/ml-langchain.yml +++ b/.github/workflows/ml-langchain.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/.github/workflows/ml-mlflow.yml b/.github/workflows/ml-mlflow.yml index ef0d447e..000f60ff 100644 --- a/.github/workflows/ml-mlflow.yml +++ b/.github/workflows/ml-mlflow.yml @@ -46,6 +46,8 @@ jobs: ports: - 4200:4200 - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g steps: From 87c23f360a630586fc7489444eb0852be8e1661f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 7 Feb 2024 21:05:22 +0100 Subject: [PATCH 3/4] CI: Align GHA recipes about CrateDB container configuration --- .github/workflows/framework-apache-superset.yml | 11 ++++++++--- .github/workflows/lang-java-jooq.yml | 2 +- .github/workflows/lang-java-maven.yml | 2 +- .github/workflows/lang-php-amphp.yml | 2 +- .github/workflows/lang-php-pdo.yml | 2 +- .github/workflows/lang-ruby.yml | 2 +- .github/workflows/ml-automl.yml | 2 +- .github/workflows/ml-mlflow.yml | 2 +- .github/workflows/stack-kafka-flink.yml | 2 +- 9 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/framework-apache-superset.yml b/.github/workflows/framework-apache-superset.yml index 5217c008..429e9016 100644 --- a/.github/workflows/framework-apache-superset.yml +++ b/.github/workflows/framework-apache-superset.yml @@ -28,7 +28,12 @@ concurrency: jobs: - tests: + test: + name: " + Superset: ${{ matrix.superset-version }} + Python: ${{ matrix.python-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: @@ -37,17 +42,17 @@ jobs: os: [ ubuntu-22.04 ] superset-version: [ "2.*", "3.*" ] python-version: [ "3.11" ] + cratedb-version: [ 'nightly' ] services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 env: CRATE_HEAP_SIZE: 4g - name: Superset ${{ matrix.superset-version }}, Python ${{ matrix.python-version }} steps: - name: Acquire sources diff --git a/.github/workflows/lang-java-jooq.yml b/.github/workflows/lang-java-jooq.yml index a312ffad..d2c4688f 100644 --- a/.github/workflows/lang-java-jooq.yml +++ b/.github/workflows/lang-java-jooq.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/lang-java-maven.yml b/.github/workflows/lang-java-maven.yml index 3147b526..2e83ee88 100644 --- a/.github/workflows/lang-java-maven.yml +++ b/.github/workflows/lang-java-maven.yml @@ -44,7 +44,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/lang-php-amphp.yml b/.github/workflows/lang-php-amphp.yml index 811e9bc4..aa239a15 100644 --- a/.github/workflows/lang-php-amphp.yml +++ b/.github/workflows/lang-php-amphp.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/lang-php-pdo.yml b/.github/workflows/lang-php-pdo.yml index 621080a3..a3a8db93 100644 --- a/.github/workflows/lang-php-pdo.yml +++ b/.github/workflows/lang-php-pdo.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/lang-ruby.yml b/.github/workflows/lang-ruby.yml index 0ebf6c5e..376f6902 100644 --- a/.github/workflows/lang-ruby.yml +++ b/.github/workflows/lang-ruby.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/ml-automl.yml b/.github/workflows/ml-automl.yml index b220e71e..3b40735d 100644 --- a/.github/workflows/ml-automl.yml +++ b/.github/workflows/ml-automl.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/ml-mlflow.yml b/.github/workflows/ml-mlflow.yml index 000f60ff..3e341336 100644 --- a/.github/workflows/ml-mlflow.yml +++ b/.github/workflows/ml-mlflow.yml @@ -42,7 +42,7 @@ jobs: services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: - 4200:4200 - 5432:5432 diff --git a/.github/workflows/stack-kafka-flink.yml b/.github/workflows/stack-kafka-flink.yml index a5f823bd..69f8abff 100644 --- a/.github/workflows/stack-kafka-flink.yml +++ b/.github/workflows/stack-kafka-flink.yml @@ -28,7 +28,7 @@ concurrency: jobs: - tests: + test: runs-on: ${{ matrix.os }} strategy: From 45e06fe5394a936d9fa3960ebe898716393784fc Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 7 Feb 2024 21:05:34 +0100 Subject: [PATCH 4/4] CI: Add Apache Superset to Dependabot configuration --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3ba6eaf8..97d01c3d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -73,6 +73,13 @@ updates: schedule: interval: "weekly" + # Frameworks. + + - directory: "/framework/apache-superset" + package-ecosystem: "pip" + schedule: + interval: "weekly" + # Topics. - directory: "/topic/machine-learning/automl"