Merge pull request #972 from cescoffier/ci-rework

Refactor CI workflows (only pull requests)

.github/workflows/build-pull-request.yml

@@ -20,12 +20,15 @@ defaults:
     shell: bash
 
 jobs:
-  # Build the project, no native tests.
-  build-and-test-jvm:
-    name: Main Build
+  # Quick build (no test, Java 17)
+  # Artifacts will be reused in the other jobs
+  quick-build:
+    name: Quick Build
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      all_modules: ${{ steps.matrices.outputs.all_modules }}
+      it_modules: ${{ steps.matrices.outputs.it_modules }}
+      in_process_embedding_modules: ${{ steps.matrices.outputs.in_process_embedding_modules }}
 
     steps:
       - uses: actions/checkout@v4
@@ -38,44 +41,62 @@ jobs:
           cache: 'maven'
 
       - name: Build with Maven
-        run: ./mvnw -B clean install -Dno-format -ntp
+        run: ./mvnw -B clean install -DskipTests -Dno-format -ntp
 
       - name: Zip the Maven repo
         run: |
           tar -czf ${{ runner.temp }}/maven-repo.tgz -C ~ .m2/repository
           # Avoid caching our own artifacts
           rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
+
       - name: Persist the Maven repo
         uses: actions/upload-artifact@v4
         with:
           name: maven-repo
           path: ${{ runner.temp }}/maven-repo.tgz
           retention-days: 5
 
-      - name: Output the matrix
-        id: set-matrix
+      - name: Compute matrices
+        id: matrices
         run: |
+          # Compute the JVM tests 
+          ALL_MODULES=$(find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
+                | sed 's|^\./||' \
+                | sort -u \
+                | jq -R -s -c 'split("\n")[:-1]')
+          
+          # Integration tests (without the in-process embedding models)
+          # Remove JLama from the list
           cd integration-tests
-          # skip RAG module as it doesn't have any native-compatible tests now
-          MATRIX='{"testModule":'$( \
+          IT_MODULES=$( \
               find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
                 | sed 's|^\./||' \
                 | sort -u \
-                | grep -v rag \
                 | grep -v jlama \
-                | jq -R -s -c 'split("\n")[:-1]' \
-          )'}'
-          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
-
-  # Test the project with different JDKs.
-  test-jvm-alt:
+                | grep -v in-process-embedding-models \
+                | jq -R -s -c 'split("\n")[:-1]')
+          
+          # We extract in process embedding models as there are many modules and we want parallelism 
+          cd in-process-embedding-models
+          IN_PROCESS_EMBEDDING_MODULES=$( \
+              find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
+                | sed 's|^\./||' \
+                | sort -u \
+                | jq -R -s -c 'split("\n")[:-1]')
+                    
+          echo "all_modules=${ALL_MODULES}" >> $GITHUB_OUTPUT
+          echo "it_modules=${IT_MODULES}" >> $GITHUB_OUTPUT
+          echo "in_process_embedding_modules=${IN_PROCESS_EMBEDDING_MODULES}" >> $GITHUB_OUTPUT
+
+  test-jvm:
+    needs: quick-build
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        java: [21, 22, 23]
-    name: Test on ${{ matrix.os }} - ${{ matrix.java }}
-    runs-on: ${{ matrix.os }}
+        java: [ 17, 21, 23 ]
+        module: ${{fromJson(needs.quick-build.outputs.all_modules)}}
+    name: Build and Test ${{ matrix.module }} on Java ${{ matrix.java }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -84,23 +105,68 @@ jobs:
         with:
           distribution: temurin
           java-version: ${{ matrix.java }}
-          cache: 'maven'
 
-      - name: Build with Maven
-        run: ./mvnw -B clean install -Dno-format -ntp
+      - name: Download the Maven repo
+        uses: actions/download-artifact@v4
+        with:
+          name: maven-repo
+          path: ..
+      - name: Unzip the Maven Repo
+        shell: bash
+        run: |
+          tar -xzf ../maven-repo.tgz -C ~
 
-      - name: Avoid caching our own artifacts
+      # Build Jlama if JDK >= 21
+      # It's not build by default as it requires Java 21+
+      - name: Build JLama extension
+        if: ${{ matrix.java >= 21 }}
         run: |
-          rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
+          ./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/jlama/pom.xml
+
+      - name: Run tests of ${{ matrix.module }} with JDK ${{ matrix.java }}
+        run: |
+          cd ${{ matrix.module }} && ../mvnw -B verify -Dci=true -Dno-format -ntp    
+
 
   native-tests:
-    needs: build-and-test-jvm
-    name: ${{matrix.testModule}} native tests
+    needs: quick-build
+    name: Native tests ${{matrix.module}}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.build-and-test-jvm.outputs.matrix) }}
+      matrix:
+        module: ${{fromJson(needs.quick-build.outputs.it_modules)}}
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Download the Maven repo
+        uses: actions/download-artifact@v4
+        with:
+          name: maven-repo
+          path: ..
+      - name: Unzip the Maven Repo
+        shell: bash
+        run: |
+          tar -xzf ../maven-repo.tgz -C ~
 
+      - name: Run integration tests ${{matrix.module}}
+        run: |
+          cd integration-tests/${{matrix.module}} && ../../mvnw -B verify -Dnative -Dci=true -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
+
+  in-process-embedding-model-tests:
+    needs: quick-build
+    name: Native tests ${{matrix.module}}
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{fromJson(needs.quick-build.outputs.in_process_embedding_modules)}}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -120,6 +186,7 @@ jobs:
         run: |
           tar -xzf ../maven-repo.tgz -C ~
 
-      - name: Run integration test ${{matrix.testModule}}
+      - name: Run integration tests ${{matrix.module}}
         run: |
-          cd integration-tests/${{matrix.testModule}} && ../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
+          cd integration-tests/in-process-embedding-models/${{matrix.module}}
+          ../../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dci=true -Dnative.surefire.skip -Dno-format -ntp

docs/modules/ROOT/pages/includes/attributes.adoc

@@ -1,3 +1,3 @@
 :project-version: 0.20.3
 :langchain4j-version: 0.35.0
-:examples-dir: ./../examples/
+:examples-dir: ./../examples/

embedding-stores/pom.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-embedding-stores-parent</artifactId>
+    <name>Quarkus LangChain4j - Embedding Stores - Parent</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>chroma</module>
+        <module>infinispan</module>
+        <module>milvus</module>
+        <module>neo4j</module>
+        <module>pgvector</module>
+        <module>pinecone</module>
+        <module>qdrant</module>
+        <module>redis</module>
+    </modules>
+
+</project>

memory-stores/pom.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-memory-stores-parent</artifactId>
+    <name>Quarkus LangChain4j - Memory Stores - Parent</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>memory-store-redis</module>
+    </modules>
+
+</project>

model-auth-providers/pom.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-model-auth-providers-parent</artifactId>
+    <name>Quarkus LangChain4j - Model Auth Providers - Parent</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>oidc-model-auth-provider</module>
+    </modules>
+
+</project>

model-providers/pom.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-model-providers-parent</artifactId>
+    <name>Quarkus LangChain4j - Model Providers - Parent</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>anthropic</module>
+        <module>cohere</module>
+        <module>hugging-face</module>
+        <module>mistral</module>
+        <module>ollama</module>
+        <!-- Add another pom -->
+        <module>openai/azure-openai</module>
+        <module>openai/openai-common</module>
+        <module>openai/openai-vanilla</module>
+        <module>openai/testing-internal</module>
+
+        <module>openshift-ai</module>
+        <module>vertex-ai</module>
+        <module>vertex-ai-gemini</module>
+        <module>watsonx</module>
+    </modules>
+
+    <profiles>
+        <profile>
+            <id>jdk21-plus</id>
+            <activation>
+                <jdk>[21,)</jdk>
+            </activation>
+            <modules>
+                <module>jlama</module>
+            </modules>
+        </profile>
+    </profiles>
+
+
+</project>

pom.xml

@@ -13,42 +13,13 @@
   <name>Quarkus LangChain4j - Parent</name>
   <modules>
     <module>core</module>
-
-    <module>embedding-stores/chroma</module>
-
-    <module>embedding-stores/infinispan</module>
-    <module>embedding-stores/milvus</module>
-    <module>embedding-stores/neo4j</module>
-    <module>embedding-stores/pgvector</module>
-    <module>embedding-stores/pinecone</module>
-    <module>embedding-stores/qdrant</module>
-    <module>embedding-stores/redis</module>
-    <module>memory-stores/memory-store-redis</module>
-
-    <module>model-providers/anthropic</module>
-
-    <module>model-providers/cohere</module>
-    <module>model-providers/hugging-face</module>
-    <module>model-providers/mistral</module>
-    <module>model-providers/ollama</module>
-    <module>model-providers/openai/testing-internal</module>
-    <module>model-providers/openai/azure-openai</module>
-    <module>model-providers/openai/openai-common</module>
-    <module>model-providers/openai/openai-vanilla</module>
-    <module>model-providers/openshift-ai</module>
-    <module>model-providers/vertex-ai</module>
-    <module>model-providers/vertex-ai-gemini</module>
-    <module>model-providers/watsonx</module>
-
-    <module>model-auth-providers/oidc-model-auth-provider</module>
-
-    <module>quarkus-integrations/websockets-next</module>
-
-    <module>web-search-engines/tavily</module>
-
-    <module>rag/easy-rag</module>
-
-    <module>rag/parsers-base</module>
+    <module>embedding-stores</module>
+    <module>memory-stores</module>
+    <module>model-auth-providers</module>
+    <module>model-providers</module>
+    <module>quarkus-integrations</module>
+    <module>rag</module>
+    <module>tools</module>
     <module>testing-internal</module>
   </modules>
   <scm>
@@ -208,26 +179,9 @@
         </property>
       </activation>
       <modules>
-        <module>samples/email-a-poem</module>
-        <module>samples/cli-translator</module>
-        <module>samples/review-triage</module>
-        <module>samples/fraud-detection</module>
-        <module>samples/secure-fraud-detection</module>
-        <module>samples/secure-vertex-ai-gemini-poem</module>
-        <module>samples/chatbot</module>
-        <module>samples/chatbot-easy-rag</module>
-        <module>samples/sql-chatbot</module>
-        <module>samples/chatbot-web-search</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>jdk21-plus</id>
-      <activation>
-        <jdk>[21,)</jdk>
-      </activation>
-      <modules>
-        <module>model-providers/jlama</module>
+        <module>samples</module>
       </modules>
     </profile>
+
   </profiles>
 </project>

quarkus-integrations/pom.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-integrations-parent</artifactId>
+    <name>Quarkus LangChain4j - Quarkus Integrations - Parent</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>websockets-next</module>
+    </modules>
+
+</project>