Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MLOB] add LLM obs configs #8076

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+212 −11
Open

[MLOB] add LLM obs configs #8076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a8108c6
add LLM obs configs
gary-huang Sep 4, 2024
5259c28
refactor & centralize api key checks
gary-huang Dec 31, 2024
36b653b
rename LLM_OBS consts to be more aligned with other languages
gary-huang Jan 2, 2025
14f71b3
undo jmx fetch changes
gary-huang Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,9 @@ public static Set<InstrumenterModule.TargetSystem> getEnabledSystems() {
if (cfg.isUsmEnabled()) {
enabledSystems.add(InstrumenterModule.TargetSystem.USM);
}
if (cfg.isLlmObsEnabled()) {
enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS);
}
return enabledSystems;
}

Expand Down
3 changes: 2 additions & 1 deletion ...ava-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ public enum TargetSystem {
APPSEC,
IAST,
CIVISIBILITY,
USM
USM,
LLMOBS,
}

private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class);
Expand Down
3 changes: 3 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ public final class ConfigDefaults {

static final boolean DEFAULT_IAST_STACK_TRACE_ENABLED = true;

static final boolean DEFAULT_LLM_OBS_ENABLED = false;
static final boolean DEFAULT_LLM_OBS_AGENTLESS_ENABLED = false;

static final boolean DEFAULT_USM_ENABLED = false;

static final boolean DEFAULT_CIVISIBILITY_ENABLED = false;
Expand Down
16 changes: 16 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package datadog.trace.api.config;

/**
* Constant with names of configuration options for LLM Observability. (EXPERIMENTAL AND SUBJECT TO
* CHANGE)
*/
public final class LlmObsConfig {

public static final String LLMOBS_ENABLED = "llmobs.enabled";

public static final String LLMOBS_ML_APP = "llmobs.ml.app";

public static final String LLMOBS_AGENTLESS_ENABLED = "llmobs.agentless.enabled";

private LlmObsConfig() {}
}
65 changes: 55 additions & 10 deletions internal-api/src/main/java/datadog/trace/api/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME;
import static datadog.trace.api.config.IastConfig.*;
import static datadog.trace.api.config.JmxFetchConfig.*;
import static datadog.trace.api.config.LlmObsConfig.*;
import static datadog.trace.api.config.ProfilingConfig.*;
import static datadog.trace.api.config.RemoteConfigConfig.*;
import static datadog.trace.api.config.TraceInstrumentationConfig.*;
Expand Down Expand Up @@ -309,6 +310,9 @@ public static String getHostName() {
private final boolean iastExperimentalPropagationEnabled;
private final String iastSecurityControlsConfiguration;

private final boolean llmObsAgentlessEnabled;
private final String llmObsMlApp;

private final boolean ciVisibilityTraceSanitationEnabled;
private final boolean ciVisibilityAgentlessEnabled;
private final String ciVisibilityAgentlessUrl;
Expand Down Expand Up @@ -1336,6 +1340,10 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
iastSecurityControlsConfiguration =
configProvider.getString(IAST_SECURITY_CONTROLS_CONFIGURATION, null);

llmObsAgentlessEnabled =
configProvider.getBoolean(LLMOBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED);
llmObsMlApp = configProvider.getString(LLMOBS_ML_APP);

ciVisibilityTraceSanitationEnabled =
configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true);

Expand Down Expand Up @@ -1766,21 +1774,46 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
this.traceFlushIntervalSeconds =
configProvider.getFloat(
TracerConfig.TRACE_FLUSH_INTERVAL, ConfigDefaults.DEFAULT_TRACE_FLUSH_INTERVAL);
if (profilingAgentless && apiKey == null) {
log.warn(
"Agentless profiling activated but no api key provided. Profile uploading will likely fail");
}

this.tracePostProcessingTimeout =
configProvider.getLong(
TRACE_POST_PROCESSING_TIMEOUT, ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT);

if (isCiVisibilityEnabled()
&& ciVisibilityAgentlessEnabled
&& (apiKey == null || apiKey.isEmpty())) {
throw new FatalAgentMisconfigurationError(
"Attempt to start in Agentless mode without API key. "
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
if (isLlmObsEnabled()) {
log.debug("Attempting to enable LLM Observability");
Yun-Kim marked this conversation as resolved.
Show resolved Hide resolved
if (llmObsMlApp == null || llmObsMlApp.isEmpty()) {
throw new IllegalArgumentException(
"Attempt to enable LLM Observability without ML app defined."
+ "Please ensure that the name of the ML app is provided through properties or env variable");
}

log.debug(
"LLM Observability enabled for ML app {}, agentless mode {}",
llmObsMlApp,
llmObsAgentlessEnabled);
}

// if API key is not provided, check if any products are using agentless mode and require it
if (apiKey == null || apiKey.isEmpty()) {
// CI Visibility
if (isCiVisibilityEnabled() && ciVisibilityAgentlessEnabled) {
throw new FatalAgentMisconfigurationError(
"Attempt to start in CI Visibility in Agentless mode without API key. "
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
}

// Profiling
if (profilingAgentless) {
log.warn(
"Agentless profiling activated but no api key provided. Profile uploading will likely fail");
}

// LLM Observability
if (isLlmObsEnabled() && llmObsAgentlessEnabled) {
throw new FatalAgentMisconfigurationError(
"Attempt to start LLM Observability in Agentless mode without API key. "
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
}
}

this.telemetryDebugRequestsEnabled =
Expand Down Expand Up @@ -2640,6 +2673,18 @@ public String getIastSecurityControlsConfiguration() {
return iastSecurityControlsConfiguration;
}

public boolean isLlmObsEnabled() {
return instrumenterConfig.isLlmObsEnabled();
}

public boolean isLlmObsAgentlessEnabled() {
return llmObsAgentlessEnabled;
}

public String getLlmObsMlApp() {
return llmObsMlApp;
Comment on lines +2680 to +2685
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these two options not set on the instrumenterConfig object? We're fine to just store this directly on the Config instance?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think it is fine to store these in the config instance for now, i am not sure but it is possible we may need them, instrumenterConfig seems to be about starting actual implementations classes if a product is enabled, i think we can add them later as needed

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup that's fine with me to add them later when needed. Thanks for clarifying

}

public boolean isCiVisibilityEnabled() {
return instrumenterConfig.isCiVisibilityEnabled();
}
Expand Down
9 changes: 9 additions & 0 deletions internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_MEASURE_METHODS;
import static datadog.trace.api.ConfigDefaults.DEFAULT_RESOLVER_RESET_INTERVAL;
import static datadog.trace.api.ConfigDefaults.DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION;
Expand All @@ -26,6 +27,7 @@
import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE;
import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER;
import static datadog.trace.api.config.IastConfig.IAST_ENABLED;
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED;
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED;
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED_DEFAULT;
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED;
Expand Down Expand Up @@ -111,6 +113,7 @@ public class InstrumenterConfig {
private final boolean iastFullyDisabled;
private final boolean usmEnabled;
private final boolean telemetryEnabled;
private final boolean llmObsEnabled;

private final String traceExtensionsPath;

Expand Down Expand Up @@ -199,6 +202,7 @@ private InstrumenterConfig() {
iastFullyDisabled = iastEnabled != null && !iastEnabled;
usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED);
telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED);
llmObsEnabled = configProvider.getBoolean(LLMOBS_ENABLED, DEFAULT_LLM_OBS_ENABLED);
} else {
// disable these features in native-image
ciVisibilityEnabled = false;
Expand All @@ -207,6 +211,7 @@ private InstrumenterConfig() {
iastFullyDisabled = true;
telemetryEnabled = false;
usmEnabled = false;
llmObsEnabled = false;
}

traceExtensionsPath = configProvider.getString(TRACE_EXTENSIONS_PATH);
Expand Down Expand Up @@ -355,6 +360,10 @@ public boolean isIastFullyDisabled() {
return iastFullyDisabled;
}

public boolean isLlmObsEnabled() {
return llmObsEnabled;
}

public boolean isUsmEnabled() {
return usmEnabled;
}
Expand Down
124 changes: 124 additions & 0 deletions internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_REFRESH_BEANS_PE
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_HOST
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_PORT
import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_AGENTLESS_ENABLED
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ML_APP
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED
import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_OLD
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_VERY_OLD
Expand Down Expand Up @@ -163,6 +166,9 @@ class ConfigTest extends DDSpecification {
private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS"
private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD"
private static final DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH"
private static final DD_LLMOBS_ENABLED_ENV = "DD_LLMOBS_ENABLED"
private static final DD_LLMOBS_ML_APP_ENV = "DD_LLMOBS_ML_APP"
private static final DD_LLMOBS_AGENTLESS_ENABLED_ENV = "DD_LLMOBS_AGENTLESS_ENABLED"

def "specify overrides via properties"() {
setup:
Expand Down Expand Up @@ -2208,6 +2214,124 @@ class ConfigTest extends DDSpecification {
!hostname.trim().isEmpty()
}

def "config instantiation should fail if llm obs is enabled via sys prop and ml app is not set"() {
setup:
Properties properties = new Properties()
properties.setProperty(LLMOBS_ENABLED, "true")

when:
new Config(ConfigProvider.withPropertiesOverride(properties))

then:
thrown IllegalArgumentException
}

def "config instantiation should fail if llm obs is enabled via env var and ml app is not set"() {
setup:
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")

when:
new Config()

then:
thrown IllegalArgumentException
}


def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via sys prop and ml app is set"() {
setup:
Properties properties = new Properties()
properties.setProperty(LLMOBS_ENABLED, "true")
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "false")
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")

when:
def config = new Config(ConfigProvider.withPropertiesOverride(properties))

then:
noExceptionThrown()
config.isLlmObsEnabled()
!config.isLlmObsAgentlessEnabled()
config.llmObsMlApp == "test-ml-app"
}

def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via env var and ml app is set"() {
setup:
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "test-ml-app")

when:
def config = new Config()

then:
noExceptionThrown()
config.isLlmObsEnabled()
!config.isLlmObsAgentlessEnabled()
config.llmObsMlApp == "test-ml-app"
}

def "config instantiation should fail if llm obs is in agentless mode via sys prop and API key is not set"() {
setup:
Properties properties = new Properties()
properties.setProperty(LLMOBS_ENABLED, "true")
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true")
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")

when:
new Config(ConfigProvider.withPropertiesOverride(properties))

then:
thrown FatalAgentMisconfigurationError
}

def "config instantiation should fail if llm obs is in agentless mode via env var and API key is not set"() {
setup:
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")

when:
new Config()

then:
thrown FatalAgentMisconfigurationError
}

def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via sys prop"() {
setup:
Properties properties = new Properties()
properties.setProperty(LLMOBS_ENABLED, "true")
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true")
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")
properties.setProperty(API_KEY, "123456789")

when:
def config = new Config(ConfigProvider.withPropertiesOverride(properties))

then:
noExceptionThrown()
config.isLlmObsEnabled()
config.isLlmObsAgentlessEnabled()
config.llmObsMlApp == "test-ml-app"
}

def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via env var"() {
setup:
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")
environmentVariables.set(DD_API_KEY_ENV, "8663294466")

when:
def config = new Config()

then:
noExceptionThrown()
config.isLlmObsEnabled()
config.isLlmObsAgentlessEnabled()
config.llmObsMlApp == "a"
}

def "config instantiation should fail if CI visibility agentless mode is enabled and API key is not set"() {
setup:
Properties properties = new Properties()
Expand Down
Loading