Skip to content

Commit

Permalink
Introduce llama3-java module
Browse files Browse the repository at this point in the history
  • Loading branch information
@geoand
geoand committed Oct 24, 2024
1 parent 7279fe5 commit 4ee18f5
Show file tree
Hide file tree
Showing 25 changed files with 3,585 additions and 0 deletions.
56 changes: 56 additions & 0 deletions model-providers/llama3-java/deployment/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.quarkiverse.langchain4j</groupId>
<artifactId>quarkus-langchain4j-llama3-java-parent</artifactId>
<version>999-SNAPSHOT</version>
</parent>
<artifactId>quarkus-langchain4j-llama3-java-deployment</artifactId>
<name>Quarkus LangChain4j - Llama3 - Java - Deployment</name>
<dependencies>
<dependency>
<groupId>io.quarkiverse.langchain4j</groupId>
<artifactId>quarkus-langchain4j-llama3-java</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.quarkiverse.langchain4j</groupId>
<artifactId>quarkus-langchain4j-core-deployment</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-internal</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.quarkiverse.langchain4j</groupId>
<artifactId>quarkus-langchain4j-testing-internal</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
</plugins>
</build>
</project>
16 changes: 16 additions & 0 deletions ...ment/src/main/java/io/quarkiverse/langchain4j/llama3/deployment/ChatModelBuildConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package io.quarkiverse.langchain4j.llama3.deployment;

import java.util.Optional;

import io.quarkus.runtime.annotations.ConfigDocDefault;
import io.quarkus.runtime.annotations.ConfigGroup;

@ConfigGroup
public interface ChatModelBuildConfig {

/**
* Whether the model should be enabled
*/
@ConfigDocDefault("true")
Optional<Boolean> enabled();
}
25 changes: 25 additions & 0 deletions ...in/java/io/quarkiverse/langchain4j/llama3/deployment/LangChain4jJlamaBuildTimeConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package io.quarkiverse.langchain4j.llama3.deployment;

import static io.quarkus.runtime.annotations.ConfigPhase.BUILD_TIME;

import io.quarkus.runtime.annotations.ConfigRoot;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;

@ConfigRoot(phase = BUILD_TIME)
@ConfigMapping(prefix = "quarkus.langchain4j.llama3")
public interface LangChain4jJlamaBuildTimeConfig {

/**
* Determines whether the necessary Jlama models are downloaded and included in the jar at build time.
* Currently, this option is only valid for {@code fast-jar} deployments.
*/
@WithDefault("true")
boolean includeModelsInArtifact();

/**
* Chat model related settings
*/
ChatModelBuildConfig chatModel();

}
238 changes: 238 additions & 0 deletions ...eployment/src/main/java/io/quarkiverse/langchain4j/llama3/deployment/Llama3Processor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
package io.quarkiverse.langchain4j.llama3.deployment;

import static io.quarkiverse.langchain4j.deployment.LangChain4jDotNames.CHAT_MODEL;

import java.nio.file.Path;
import java.util.List;

import jakarta.enterprise.context.ApplicationScoped;

import org.jboss.jandex.AnnotationInstance;
import org.jboss.logging.Logger;
import org.slf4j.LoggerFactory;

import io.quarkiverse.langchain4j.ModelName;
import io.quarkiverse.langchain4j.deployment.items.ChatModelProviderCandidateBuildItem;
import io.quarkiverse.langchain4j.deployment.items.SelectedChatModelProviderBuildItem;
import io.quarkiverse.langchain4j.llama3.runtime.Llama3Recorder;
import io.quarkiverse.langchain4j.llama3.runtime.config.LangChain4jLlama3FixedRuntimeConfig;
import io.quarkiverse.langchain4j.llama3.runtime.config.LangChain4jLlama3RuntimeConfig;
import io.quarkiverse.langchain4j.llama3.runtime.graal.Llama3Feature;
import io.quarkiverse.langchain4j.runtime.NamedConfigUtil;
import io.quarkus.arc.deployment.SyntheticBeanBuildItem;
import io.quarkus.builder.item.MultiBuildItem;
import io.quarkus.deployment.annotations.BuildProducer;
import io.quarkus.deployment.annotations.BuildStep;
import io.quarkus.deployment.annotations.ExecutionTime;
import io.quarkus.deployment.annotations.Record;
import io.quarkus.deployment.builditem.FeatureBuildItem;
import io.quarkus.deployment.builditem.NativeImageFeatureBuildItem;
import io.quarkus.deployment.builditem.nativeimage.NativeImageEnableModule;
import io.quarkus.deployment.builditem.nativeimage.RuntimeInitializedPackageBuildItem;

public class Llama3Processor {

private final static Logger LOGGER = Logger.getLogger(Llama3Processor.class);

private static final String FEATURE = "langchain4j-llama3-java";
private static final String PROVIDER = "llama3-java";
private static final org.slf4j.Logger log = LoggerFactory.getLogger(Llama3Processor.class);

@BuildStep
FeatureBuildItem feature() {
return new FeatureBuildItem(FEATURE);
}

@BuildStep
public void providerCandidates(BuildProducer<ChatModelProviderCandidateBuildItem> chatProducer,
LangChain4jJlamaBuildTimeConfig config) {
if (config.chatModel().enabled().isEmpty() || config.chatModel().enabled().get()) {
chatProducer.produce(new ChatModelProviderCandidateBuildItem(PROVIDER));
}
}

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@BuildStep
@Record(ExecutionTime.RUNTIME_INIT)
void generateBeans(Llama3Recorder recorder,
List<SelectedChatModelProviderBuildItem> selectedChatItem,
LangChain4jLlama3RuntimeConfig runtimeConfig,
LangChain4jLlama3FixedRuntimeConfig fixedRuntimeConfig,
BuildProducer<SyntheticBeanBuildItem> beanProducer) {

for (var selected : selectedChatItem) {
if (PROVIDER.equals(selected.getProvider())) {
String configName = selected.getConfigName();
var builder = SyntheticBeanBuildItem.configure(CHAT_MODEL).setRuntimeInit().defaultBean()
.scope(ApplicationScoped.class)
.supplier(recorder.chatModel(runtimeConfig, fixedRuntimeConfig, configName));
addQualifierIfNecessary(builder, configName);
beanProducer.produce(builder.done());
}
}
}

private void addQualifierIfNecessary(SyntheticBeanBuildItem.ExtendedBeanConfigurator builder, String configName) {
if (!NamedConfigUtil.isDefault(configName)) {
builder.addQualifier(AnnotationInstance.builder(ModelName.class).add("value", configName).build());
}
}

@BuildStep
public void nativeSupport(BuildProducer<RuntimeInitializedPackageBuildItem> runtimeInitializedPackageProducer,
BuildProducer<NativeImageEnableModule> moduleProducer,
BuildProducer<NativeImageFeatureBuildItem> nativeFeatureProducer) {
runtimeInitializedPackageProducer
.produce(new RuntimeInitializedPackageBuildItem("io.quarkiverse.langchain4j.llama3.copy"));
moduleProducer.produce(new NativeImageEnableModule("jdk.incubator.vector"));
nativeFeatureProducer.produce(new NativeImageFeatureBuildItem(Llama3Feature.class));
}

// @Produce(ServiceStartBuildItem.class)
// @BuildStep
// void downloadModels(List<SelectedChatModelProviderBuildItem> selectedChatModels,
// List<SelectedEmbeddingModelCandidateBuildItem> selectedEmbeddingModels,
// LoggingSetupBuildItem loggingSetupBuildItem,
// Optional<ConsoleInstalledBuildItem> consoleInstalledBuildItem,
// LaunchModeBuildItem launchMode,
// LangChain4jJlamaBuildTimeConfig buildTimeConfig,
// LangChain4jJlamaFixedRuntimeConfig fixedRuntimeConfig,
// BuildProducer<ModelDownloadedBuildItem> modelDownloadedProducer) {
// if (!buildTimeConfig.includeModelsInArtifact()) {
// return;
// }
// JlamaModelRegistry registry = JlamaModelRegistry.getOrCreate(fixedRuntimeConfig.modelsPath());
//
// BigDecimal ONE_HUNDRED = new BigDecimal("100");
//
// if (buildTimeConfig.chatModel().enabled().orElse(true) || buildTimeConfig.embeddingModel().enabled().orElse(true)) {
// List<String> modelsNeeded = new ArrayList<>();
// for (var selected : selectedChatModels) {
// if (PROVIDER.equals(selected.getProvider())) {
// String configName = selected.getConfigName();
//
// String modelName = NamedConfigUtil.isDefault(configName)
// ? fixedRuntimeConfig.defaultConfig().chatModel().modelName()
// : fixedRuntimeConfig.namedConfig().get(configName).chatModel().modelName();
// modelsNeeded.add(modelName);
// }
// }
//
// for (var selected : selectedEmbeddingModels) {
// if (PROVIDER.equals(selected.getProvider())) {
// String configName = selected.getConfigName();
//
// String modelName = NamedConfigUtil.isDefault(configName)
// ? fixedRuntimeConfig.defaultConfig().embeddingModel().modelName()
// : fixedRuntimeConfig.namedConfig().get(configName).embeddingModel().modelName();
// modelsNeeded.add(modelName);
// }
// }
//
// if (!modelsNeeded.isEmpty()) {
// StartupLogCompressor compressor = new StartupLogCompressor(
// (launchMode.isTest() ? "(test) " : "") + "Jlama model pull:",
// consoleInstalledBuildItem,
// loggingSetupBuildItem);
//
// for (String modelName : modelsNeeded) {
// JlamaModelRegistry.ModelInfo modelInfo = JlamaModelRegistry.ModelInfo.from(modelName);
// Path pathOfModelDirOnDisk = SafeTensorSupport.constructLocalModelPath(
// registry.getModelCachePath().toAbsolutePath().toString(), modelInfo.owner(),
// modelInfo.name());
// // Check if the model is already downloaded
// // this is done automatically by download model, but we want to provide a good progress experience, so we do it again here
// if (Files.exists(pathOfModelDirOnDisk.resolve(".finished"))) {
// LOGGER.debug("Model " + modelName + "already exists in " + pathOfModelDirOnDisk);
// } else {
// // we pull one model at a time and provide progress updates to the user via logging
// LOGGER.info("Pulling model " + modelName);
//
// try {
// registry.downloadModel(modelName, Optional.empty(), Optional.of(new ProgressReporter() {
// @Override
// public void update(String filename, long sizeDownloaded, long totalSize) {
// // Jlama downloads a bunch of files for each mode of which only the weights file is large
// // and makes sense to report progress on
// if (totalSize < 100_000) {
// return;
// }
//
// BigDecimal percentage = new BigDecimal(sizeDownloaded).divide(new BigDecimal(totalSize), 4,
// RoundingMode.HALF_DOWN).multiply(ONE_HUNDRED);
// BigDecimal progress = percentage.setScale(2, RoundingMode.HALF_DOWN);
// if (progress.compareTo(ONE_HUNDRED) >= 0) {
// // avoid showing 100% for too long
// LOGGER.infof("Verifying and cleaning up\n", progress);
// } else {
// LOGGER.infof("Progress: %s%%\n", progress);
// }
// }
// }));
// } catch (IOException e) {
// compressor.closeAndDumpCaptured();
// throw new UncheckedIOException(e);
// }
// }
//
// modelDownloadedProducer.produce(new ModelDownloadedBuildItem(modelName, pathOfModelDirOnDisk));
// }
//
// compressor.close();
// }
// }
//
// }

/**
* When building a fast jar, we can copy the model files into the directory
*
*/
// @BuildStep(onlyIf = IsNormal.class)
// @Produce(ArtifactResultBuildItem.class)
// public void copyToFastJar(List<ModelDownloadedBuildItem> models,
// Optional<JarBuildItem> jarBuildItem) {
// if (!jarBuildItem.isPresent()) {
// return;
// }
//
// Path jarPath = jarBuildItem.get().getPath();
// if (!JarResultBuildStep.QUARKUS_RUN_JAR.equals(jarPath.getFileName().toString())) {
// return;
// }
//
// Path quarkusAppDir = jarPath.getParent();
// Path jlamaInQuarkusAppDir = quarkusAppDir.resolve("jlama");
//
// for (ModelDownloadedBuildItem bi : models) {
// try {
// JlamaModelRegistry.ModelInfo modelInfo = JlamaModelRegistry.ModelInfo.from(bi.getModelName());
// Path targetDir = jlamaInQuarkusAppDir.resolve(modelInfo.toFileName());
// Files.createDirectories(targetDir);
// PathUtils.copyDirectory(bi.getDirectory(), targetDir);
// } catch (IOException e) {
// throw new UncheckedIOException(e);
// }
// }
//
// }

public static final class ModelDownloadedBuildItem extends MultiBuildItem {

private final String modelName;
private final Path directory;

public ModelDownloadedBuildItem(String modelName, Path directory) {
this.modelName = modelName;
this.directory = directory;
}

public String getModelName() {
return modelName;
}

public Path getDirectory() {
return directory;
}
}
}
24 changes: 24 additions & 0 deletions model-providers/llama3-java/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.quarkiverse.langchain4j</groupId>
<artifactId>quarkus-langchain4j-parent</artifactId>
<version>999-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>quarkus-langchain4j-llama3-java-parent</artifactId>
<name>Quarkus LangChain4j - Llama 3 - Java - Parent</name>
<packaging>pom</packaging>

<properties>
<maven.compiler.release>21</maven.compiler.release>
</properties>

<modules>
<module>deployment</module>
<module>runtime</module>
</modules>


</project>
Loading

0 comments on commit 4ee18f5

Please sign in to comment.