diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index 7d817cfe0..f420175e8 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -20,12 +20,15 @@ defaults: shell: bash jobs: - # Build the project, no native tests. - build-and-test-jvm: - name: Main Build + # Quick build (no test, Java 17) + # Artifacts will be reused in the other jobs + quick-build: + name: Quick Build runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + all_modules: ${{ steps.matrices.outputs.all_modules }} + it_modules: ${{ steps.matrices.outputs.it_modules }} + in_process_embedding_modules: ${{ steps.matrices.outputs.in_process_embedding_modules }} steps: - uses: actions/checkout@v4 @@ -38,13 +41,14 @@ jobs: cache: 'maven' - name: Build with Maven - run: ./mvnw -B clean install -Dno-format -ntp + run: ./mvnw -B clean install -DskipTests -Dno-format -ntp - name: Zip the Maven repo run: | tar -czf ${{ runner.temp }}/maven-repo.tgz -C ~ .m2/repository # Avoid caching our own artifacts rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j + - name: Persist the Maven repo uses: actions/upload-artifact@v4 with: @@ -52,30 +56,47 @@ jobs: path: ${{ runner.temp }}/maven-repo.tgz retention-days: 5 - - name: Output the matrix - id: set-matrix + - name: Compute matrices + id: matrices run: | + # Compute the JVM tests + ALL_MODULES=$(find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \ + | sed 's|^\./||' \ + | sort -u \ + | jq -R -s -c 'split("\n")[:-1]') + + # Integration tests (without the in-process embedding models) + # Remove JLama from the list cd integration-tests - # skip RAG module as it doesn't have any native-compatible tests now - MATRIX='{"testModule":'$( \ + IT_MODULES=$( \ find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \ | sed 's|^\./||' \ | sort -u \ - | grep -v rag \ | grep -v jlama \ - | jq -R -s -c 'split("\n")[:-1]' \ - )'}' - echo "matrix=$MATRIX" >> $GITHUB_OUTPUT - - # Test the project with different JDKs. - test-jvm-alt: + | grep -v in-process-embedding-models \ + | jq -R -s -c 'split("\n")[:-1]') + + # We extract in process embedding models as there are many modules and we want parallelism + cd in-process-embedding-models + IN_PROCESS_EMBEDDING_MODULES=$( \ + find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \ + | sed 's|^\./||' \ + | sort -u \ + | jq -R -s -c 'split("\n")[:-1]') + + echo "all_modules=${ALL_MODULES}" >> $GITHUB_OUTPUT + echo "it_modules=${IT_MODULES}" >> $GITHUB_OUTPUT + echo "in_process_embedding_modules=${IN_PROCESS_EMBEDDING_MODULES}" >> $GITHUB_OUTPUT + + test-jvm: + needs: quick-build strategy: fail-fast: false matrix: - os: [ubuntu-latest] - java: [21, 22, 23] - name: Test on ${{ matrix.os }} - ${{ matrix.java }} - runs-on: ${{ matrix.os }} + java: [ 17, 21, 23 ] + module: ${{fromJson(needs.quick-build.outputs.all_modules)}} + name: Build and Test ${{ matrix.module }} on Java ${{ matrix.java }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -84,23 +105,68 @@ jobs: with: distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' - - name: Build with Maven - run: ./mvnw -B clean install -Dno-format -ntp + - name: Download the Maven repo + uses: actions/download-artifact@v4 + with: + name: maven-repo + path: .. + - name: Unzip the Maven Repo + shell: bash + run: | + tar -xzf ../maven-repo.tgz -C ~ - - name: Avoid caching our own artifacts + # Build Jlama if JDK >= 21 + # It's not build by default as it requires Java 21+ + - name: Build JLama extension + if: ${{ matrix.java >= 21 }} run: | - rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j + ./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/jlama/pom.xml + + - name: Run tests of ${{ matrix.module }} with JDK ${{ matrix.java }} + run: | + cd ${{ matrix.module }} && ../mvnw -B verify -Dci=true -Dno-format -ntp + native-tests: - needs: build-and-test-jvm - name: ${{matrix.testModule}} native tests + needs: quick-build + name: Native tests ${{matrix.module}} strategy: fail-fast: false - matrix: ${{ fromJson(needs.build-and-test-jvm.outputs.matrix) }} + matrix: + module: ${{fromJson(needs.quick-build.outputs.it_modules)}} runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Download the Maven repo + uses: actions/download-artifact@v4 + with: + name: maven-repo + path: .. + - name: Unzip the Maven Repo + shell: bash + run: | + tar -xzf ../maven-repo.tgz -C ~ + - name: Run integration tests ${{matrix.module}} + run: | + cd integration-tests/${{matrix.module}} && ../../mvnw -B verify -Dnative -Dci=true -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp + + in-process-embedding-model-tests: + needs: quick-build + name: Native tests ${{matrix.module}} + strategy: + fail-fast: false + matrix: + module: ${{fromJson(needs.quick-build.outputs.in_process_embedding_modules)}} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -120,6 +186,7 @@ jobs: run: | tar -xzf ../maven-repo.tgz -C ~ - - name: Run integration test ${{matrix.testModule}} + - name: Run integration tests ${{matrix.module}} run: | - cd integration-tests/${{matrix.testModule}} && ../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp + cd integration-tests/in-process-embedding-models/${{matrix.module}} + ../../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dci=true -Dnative.surefire.skip -Dno-format -ntp diff --git a/docs/modules/ROOT/pages/includes/attributes.adoc b/docs/modules/ROOT/pages/includes/attributes.adoc index a6b1ec7b7..1a640423d 100644 --- a/docs/modules/ROOT/pages/includes/attributes.adoc +++ b/docs/modules/ROOT/pages/includes/attributes.adoc @@ -1,3 +1,3 @@ :project-version: 0.20.3 :langchain4j-version: 0.35.0 -:examples-dir: ./../examples/ +:examples-dir: ./../examples/ \ No newline at end of file diff --git a/embedding-stores/pom.xml b/embedding-stores/pom.xml new file mode 100644 index 000000000..fe16c6c58 --- /dev/null +++ b/embedding-stores/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-embedding-stores-parent + Quarkus LangChain4j - Embedding Stores - Parent + pom + + + chroma + infinispan + milvus + neo4j + pgvector + pinecone + qdrant + redis + + + diff --git a/memory-stores/pom.xml b/memory-stores/pom.xml new file mode 100644 index 000000000..9749bf80b --- /dev/null +++ b/memory-stores/pom.xml @@ -0,0 +1,17 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-memory-stores-parent + Quarkus LangChain4j - Memory Stores - Parent + pom + + + memory-store-redis + + + diff --git a/model-auth-providers/pom.xml b/model-auth-providers/pom.xml new file mode 100644 index 000000000..662483856 --- /dev/null +++ b/model-auth-providers/pom.xml @@ -0,0 +1,17 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-model-auth-providers-parent + Quarkus LangChain4j - Model Auth Providers - Parent + pom + + + oidc-model-auth-provider + + + diff --git a/model-providers/pom.xml b/model-providers/pom.xml new file mode 100644 index 000000000..a8f3e40ed --- /dev/null +++ b/model-providers/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-model-providers-parent + Quarkus LangChain4j - Model Providers - Parent + pom + + + anthropic + cohere + hugging-face + mistral + ollama + + openai/azure-openai + openai/openai-common + openai/openai-vanilla + openai/testing-internal + + openshift-ai + vertex-ai + vertex-ai-gemini + watsonx + + + + + jdk21-plus + + [21,) + + + jlama + + + + + + diff --git a/pom.xml b/pom.xml index 6a480dd0c..8932f2d07 100644 --- a/pom.xml +++ b/pom.xml @@ -13,42 +13,13 @@ Quarkus LangChain4j - Parent core - - embedding-stores/chroma - - embedding-stores/infinispan - embedding-stores/milvus - embedding-stores/neo4j - embedding-stores/pgvector - embedding-stores/pinecone - embedding-stores/qdrant - embedding-stores/redis - memory-stores/memory-store-redis - - model-providers/anthropic - - model-providers/cohere - model-providers/hugging-face - model-providers/mistral - model-providers/ollama - model-providers/openai/testing-internal - model-providers/openai/azure-openai - model-providers/openai/openai-common - model-providers/openai/openai-vanilla - model-providers/openshift-ai - model-providers/vertex-ai - model-providers/vertex-ai-gemini - model-providers/watsonx - - model-auth-providers/oidc-model-auth-provider - - quarkus-integrations/websockets-next - - web-search-engines/tavily - - rag/easy-rag - - rag/parsers-base + embedding-stores + memory-stores + model-auth-providers + model-providers + quarkus-integrations + rag + tools testing-internal @@ -208,26 +179,9 @@ - samples/email-a-poem - samples/cli-translator - samples/review-triage - samples/fraud-detection - samples/secure-fraud-detection - samples/secure-vertex-ai-gemini-poem - samples/chatbot - samples/chatbot-easy-rag - samples/sql-chatbot - samples/chatbot-web-search - - - - jdk21-plus - - [21,) - - - model-providers/jlama + samples + diff --git a/quarkus-integrations/pom.xml b/quarkus-integrations/pom.xml new file mode 100644 index 000000000..61f569f2a --- /dev/null +++ b/quarkus-integrations/pom.xml @@ -0,0 +1,17 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-integrations-parent + Quarkus LangChain4j - Quarkus Integrations - Parent + pom + + + websockets-next + + + diff --git a/quarkus-integrations/websockets-next/tests/src/test/java/io/quarkiverse/langchain4j/websockets/next/tests/WebSocketsNextTest.java b/quarkus-integrations/websockets-next/tests/src/test/java/io/quarkiverse/langchain4j/websockets/next/tests/WebSocketsNextTest.java index e3ca3dbe7..380d73832 100644 --- a/quarkus-integrations/websockets-next/tests/src/test/java/io/quarkiverse/langchain4j/websockets/next/tests/WebSocketsNextTest.java +++ b/quarkus-integrations/websockets-next/tests/src/test/java/io/quarkiverse/langchain4j/websockets/next/tests/WebSocketsNextTest.java @@ -25,6 +25,7 @@ import org.jboss.shrinkwrap.api.spec.JavaArchive; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.api.extension.RegisterExtension; import dev.langchain4j.data.message.ChatMessage; @@ -40,6 +41,7 @@ import io.vertx.core.Vertx; import io.vertx.core.http.WebSocketClient; +@DisabledIfSystemProperty(named = "ci", matches = "true", disabledReason = "Does not work on CI, must be investigated") public class WebSocketsNextTest extends OpenAiBaseTest { @RegisterExtension diff --git a/rag/pom.xml b/rag/pom.xml new file mode 100644 index 000000000..9c7047ac4 --- /dev/null +++ b/rag/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-rag-parent + Quarkus LangChain4j - RAG - Parent + pom + + + easy-rag + parsers-base + + + diff --git a/samples/pom.xml b/samples/pom.xml new file mode 100644 index 000000000..5ab8bc803 --- /dev/null +++ b/samples/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-samples-parent + Quarkus LangChain4j - Samples - Parent + pom + + + chatbot + chatbot-easy-rag + chatbot-web-search + cli-translator + email-a-poem + fraud-detection + review-triage + secure-fraud-detection + secure-vertex-ai-gemini-poem + sql-chatbot + + + diff --git a/tools/pom.xml b/tools/pom.xml new file mode 100644 index 000000000..5b1b5bf0d --- /dev/null +++ b/tools/pom.xml @@ -0,0 +1,17 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-parent + 999-SNAPSHOT + + quarkus-langchain4j-tools-parent + Quarkus LangChain4j - Tools - Parent + pom + + + tavily + + + diff --git a/web-search-engines/tavily/deployment/pom.xml b/tools/tavily/deployment/pom.xml similarity index 100% rename from web-search-engines/tavily/deployment/pom.xml rename to tools/tavily/deployment/pom.xml diff --git a/web-search-engines/tavily/deployment/src/main/java/io/quarkiverse/langchain4j/tavily/deployment/TavilyProcessor.java b/tools/tavily/deployment/src/main/java/io/quarkiverse/langchain4j/tavily/deployment/TavilyProcessor.java similarity index 100% rename from web-search-engines/tavily/deployment/src/main/java/io/quarkiverse/langchain4j/tavily/deployment/TavilyProcessor.java rename to tools/tavily/deployment/src/main/java/io/quarkiverse/langchain4j/tavily/deployment/TavilyProcessor.java diff --git a/web-search-engines/tavily/deployment/src/test/java/io/quarkiverse/langchain4j/tavily/test/TavilyTest.java b/tools/tavily/deployment/src/test/java/io/quarkiverse/langchain4j/tavily/test/TavilyTest.java similarity index 100% rename from web-search-engines/tavily/deployment/src/test/java/io/quarkiverse/langchain4j/tavily/test/TavilyTest.java rename to tools/tavily/deployment/src/test/java/io/quarkiverse/langchain4j/tavily/test/TavilyTest.java diff --git a/web-search-engines/tavily/pom.xml b/tools/tavily/pom.xml similarity index 100% rename from web-search-engines/tavily/pom.xml rename to tools/tavily/pom.xml diff --git a/web-search-engines/tavily/runtime/pom.xml b/tools/tavily/runtime/pom.xml similarity index 100% rename from web-search-engines/tavily/runtime/pom.xml rename to tools/tavily/runtime/pom.xml diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/QuarkusTavilyWebSearchEngine.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/QuarkusTavilyWebSearchEngine.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/QuarkusTavilyWebSearchEngine.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/QuarkusTavilyWebSearchEngine.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/SearchDepth.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/SearchDepth.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/SearchDepth.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/SearchDepth.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveData.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveData.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveData.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveData.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyClient.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyClient.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyClient.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyClient.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyConfig.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyConfig.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyConfig.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyConfig.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyRecorder.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyRecorder.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyRecorder.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyRecorder.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyResponse.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyResponse.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyResponse.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilyResponse.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchRequest.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchRequest.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchRequest.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchRequest.java diff --git a/web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchResult.java b/tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchResult.java similarity index 100% rename from web-search-engines/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchResult.java rename to tools/tavily/runtime/src/main/java/io/quarkiverse/langchain4j/tavily/runtime/TavilySearchResult.java diff --git a/web-search-engines/tavily/runtime/src/main/resources/META-INF/beans.xml b/tools/tavily/runtime/src/main/resources/META-INF/beans.xml similarity index 100% rename from web-search-engines/tavily/runtime/src/main/resources/META-INF/beans.xml rename to tools/tavily/runtime/src/main/resources/META-INF/beans.xml diff --git a/web-search-engines/tavily/runtime/src/main/resources/META-INF/quarkus-extension.yaml b/tools/tavily/runtime/src/main/resources/META-INF/quarkus-extension.yaml similarity index 100% rename from web-search-engines/tavily/runtime/src/main/resources/META-INF/quarkus-extension.yaml rename to tools/tavily/runtime/src/main/resources/META-INF/quarkus-extension.yaml diff --git a/web-search-engines/tavily/runtime/src/test/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveDataTest.java b/tools/tavily/runtime/src/test/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveDataTest.java similarity index 100% rename from web-search-engines/tavily/runtime/src/test/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveDataTest.java rename to tools/tavily/runtime/src/test/java/io/quarkiverse/langchain4j/tavily/ShadowSensitiveDataTest.java