From 8dbeb490998dad13efd9c05afef594fad23182da Mon Sep 17 00:00:00 2001 From: Kent Dong Date: Mon, 28 Oct 2024 10:00:58 +0800 Subject: [PATCH] feat: Add more functions to AI console APIs (#361) --- .../sdk/constant/BuiltInPluginName.java | 1 + .../higress/sdk/constant/CommonKey.java | 2 + .../sdk/constant/HigressConstants.java | 2 +- .../sdk/model/ai/AiModelPredicate.java | 31 +++++ .../alibaba/higress/sdk/model/ai/AiRoute.java | 4 + .../higress/sdk/model/ai/LlmProviderType.java | 2 + .../service/HigressServiceProviderImpl.java | 2 +- .../ai/AbstractLlmProviderHandler.java | 3 +- .../sdk/service/ai/AiRouteServiceImpl.java | 114 ++++++++++++++++-- .../service/ai/LlmProviderServiceImpl.java | 1 + .../service/consumer/ConsumerServiceImpl.java | 9 ++ .../resources/plugins/model-router/README.md | 64 ++++++++++ .../plugins/model-router/README_EN.md | 63 ++++++++++ .../resources/plugins/model-router/spec.yaml | 50 ++++++++ .../main/resources/plugins/plugins.properties | 1 + 15 files changed, 333 insertions(+), 16 deletions(-) create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java create mode 100644 backend/sdk/src/main/resources/plugins/model-router/README.md create mode 100644 backend/sdk/src/main/resources/plugins/model-router/README_EN.md create mode 100644 backend/sdk/src/main/resources/plugins/model-router/spec.yaml diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java index 8b8672a6..14874f4e 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java @@ -28,6 +28,7 @@ public final class BuiltInPluginName { public static final String AI_INTENT = "ai-intent"; public static final String AI_QUOTA = "ai-quota"; public static final String AI_AGENT = "ai-agent"; + public static final String MODEL_ROUTER = "model-router"; // Auth public static final String BASIC_AUTH = "basic-auth"; diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/CommonKey.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/CommonKey.java index 748eb641..5b8acf37 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/CommonKey.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/CommonKey.java @@ -29,4 +29,6 @@ public class CommonKey { public static final String AI_ROUTE = "ai-route"; public static final String AI_ROUTE_PREFIX = AI_ROUTE + Separators.DASH; + + public static final String LLM_SERVICE_NAME_PREFIX = "llm-"; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java index d7f5d443..be764c8c 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java @@ -23,5 +23,5 @@ public class HigressConstants { public static final String INTERNAL_RESOURCE_NAME_SUFFIX = ".internal"; public static final String FALLBACK_ROUTE_NAME_SUFFIX = ".fallback"; public static final String FALLBACK_FROM_HEADER = "x-higress-fallback-from"; - public static final String LLM_SERVICE_NAME_PREFIX = "llm-"; + public static final String MODEL_ROUTER_HEADER = "x-higress-llm-provider"; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java new file mode 100644 index 00000000..33812b5e --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.model.ai; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AiModelPredicate { + + private Boolean enabled; + private String prefix; + + public void validate() { + } +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java index 52dc66f6..ef146a5a 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java @@ -36,6 +36,7 @@ public class AiRoute { private String version; private List domains; private List upstreams; + private AiModelPredicate modelPredicate; private AiRouteAuthConfig authConfig; private AiRouteFallbackConfig fallbackConfig; @@ -47,6 +48,9 @@ public void validate() { throw new ValidationException("upstreams cannot be empty."); } upstreams.forEach(AiUpstream::validate); + if (modelPredicate != null){ + modelPredicate.validate(); + } if (authConfig != null){ authConfig.validate(); } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java index 73e816a1..3e4018a4 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java @@ -20,4 +20,6 @@ private LlmProviderType() { public static final String QWEN = "qwen"; public static final String OPENAI = "openai"; + + public static final String MOONSHOT = "moonshot"; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java index 4766a982..e96a9f2e 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java @@ -59,7 +59,7 @@ class HigressServiceProviderImpl implements HigressServiceProvider { llmProviderService = new LlmProviderServiceImpl(serviceSourceService, wasmPluginService, wasmPluginInstanceService); aiRouteService = new AiRouteServiceImpl(kubernetesModelConverter, kubernetesClientService, routeService, - llmProviderService, consumerService); + llmProviderService, consumerService, wasmPluginService, wasmPluginInstanceService); } @Override diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java index fb8fc4cb..05656f37 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java @@ -21,6 +21,7 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import com.alibaba.higress.sdk.constant.CommonKey; import com.alibaba.higress.sdk.constant.HigressConstants; import com.alibaba.higress.sdk.model.ai.LlmProvider; import com.alibaba.higress.sdk.model.ai.LlmProviderProtocol; @@ -126,7 +127,7 @@ public void saveConfig(LlmProvider provider, Map configurations) } protected static String generateServiceProviderName(String llmProviderName) { - return HigressConstants.LLM_SERVICE_NAME_PREFIX + llmProviderName + return CommonKey.LLM_SERVICE_NAME_PREFIX + llmProviderName + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java index 6032d71a..dc7b065a 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java @@ -15,14 +15,18 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import com.alibaba.higress.sdk.constant.BuiltInPluginName; +import com.alibaba.higress.sdk.constant.CommonKey; import com.alibaba.higress.sdk.constant.HigressConstants; import com.alibaba.higress.sdk.constant.KubernetesConstants; import com.alibaba.higress.sdk.exception.BusinessException; @@ -31,7 +35,10 @@ import com.alibaba.higress.sdk.model.CommonPageQuery; import com.alibaba.higress.sdk.model.PaginatedResult; import com.alibaba.higress.sdk.model.Route; +import com.alibaba.higress.sdk.model.WasmPlugin; +import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; +import com.alibaba.higress.sdk.model.ai.AiModelPredicate; import com.alibaba.higress.sdk.model.ai.AiRoute; import com.alibaba.higress.sdk.model.ai.AiRouteAuthConfig; import com.alibaba.higress.sdk.model.ai.AiRouteFallbackConfig; @@ -42,6 +49,8 @@ import com.alibaba.higress.sdk.model.route.RoutePredicateTypeEnum; import com.alibaba.higress.sdk.model.route.UpstreamService; import com.alibaba.higress.sdk.service.RouteService; +import com.alibaba.higress.sdk.service.WasmPluginInstanceService; +import com.alibaba.higress.sdk.service.WasmPluginService; import com.alibaba.higress.sdk.service.consumer.ConsumerService; import com.alibaba.higress.sdk.service.kubernetes.KubernetesClientService; import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter; @@ -60,6 +69,9 @@ public class AiRouteServiceImpl implements AiRouteService { private static final Map AI_ROUTE_LABEL_SELECTORS = Map.of(KubernetesConstants.Label.CONFIG_MAP_TYPE_KEY, KubernetesConstants.Label.CONFIG_MAP_TYPE_VALUE_AI_ROUTE); + private static final String MODEL_ROUTER_ENABLE_KEY = "enable"; + private static final String ADD_HEADER_KEY_KEY = "add_header_key"; + private final KubernetesModelConverter kubernetesModelConverter; private final KubernetesClientService kubernetesClientService; @@ -70,16 +82,23 @@ public class AiRouteServiceImpl implements AiRouteService { private final ConsumerService consumerService; + private final WasmPluginService wasmPluginService; + + private final WasmPluginInstanceService wasmPluginInstanceService; + private final String routeFallbackEnvoyFilterConfig; public AiRouteServiceImpl(KubernetesModelConverter kubernetesModelConverter, KubernetesClientService kubernetesClientService, RouteService routeService, - LlmProviderService llmProviderService, ConsumerService consumerService) { + LlmProviderService llmProviderService, ConsumerService consumerService, WasmPluginService wasmPluginService, + WasmPluginInstanceService wasmPluginInstanceService) { this.kubernetesModelConverter = kubernetesModelConverter; this.kubernetesClientService = kubernetesClientService; this.routeService = routeService; this.llmProviderService = llmProviderService; this.consumerService = consumerService; + this.wasmPluginService = wasmPluginService; + this.wasmPluginInstanceService = wasmPluginInstanceService; try { this.routeFallbackEnvoyFilterConfig = @@ -94,6 +113,7 @@ public AiRouteServiceImpl(KubernetesModelConverter kubernetesModelConverter, @Override public AiRoute add(AiRoute route) { + fillDefaultValues(route); V1ConfigMap configMap = kubernetesModelConverter.aiRoute2ConfigMap(route); V1ConfigMap newConfigMap; try { @@ -147,6 +167,7 @@ public void delete(String routeName) { @Override public AiRoute update(AiRoute route) { + fillDefaultValues(route); V1ConfigMap configMap = kubernetesModelConverter.aiRoute2ConfigMap(route); V1ConfigMap updatedConfigMap; try { @@ -165,12 +186,21 @@ public AiRoute update(AiRoute route) { return kubernetesModelConverter.configMap2AiRoute(updatedConfigMap); } + private void fillDefaultValues(AiRoute route) { + AiModelPredicate modelPredicate = route.getModelPredicate(); + if (modelPredicate != null && Boolean.TRUE.equals(modelPredicate.getEnabled()) + && StringUtils.isEmpty(modelPredicate.getPrefix())) { + modelPredicate.setPrefix(route.getName()); + } + } + private void writeAiRouteResources(AiRoute aiRoute) { - String routeName = aiRoute.getName() + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + String routeName = buildRouteResourceName(aiRoute.getName()); Route route = buildRoute(routeName, aiRoute); setUpstreams(route, aiRoute.getUpstreams()); saveRoute(route); writeAuthConfigResources(routeName, aiRoute.getAuthConfig()); + writeModelRouteResources(aiRoute.getModelPredicate()); } private void writeAiRouteFallbackResources(AiRoute aiRoute) { @@ -180,16 +210,18 @@ private void writeAiRouteFallbackResources(AiRoute aiRoute) { return; } - final String originalRouteName = aiRoute.getName() + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + final String originalRouteName = buildRouteResourceName(aiRoute.getName()); - final String fallbackRouteName = aiRoute.getName() + HigressConstants.FALLBACK_ROUTE_NAME_SUFFIX - + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + final String fallbackRouteName = buildFallbackRouteResourceName(aiRoute.getName()); Route route = buildRoute(fallbackRouteName, aiRoute); KeyedRoutePredicate fallbackHeaderPredicate = new KeyedRoutePredicate(HigressConstants.FALLBACK_FROM_HEADER); fallbackHeaderPredicate.setMatchType(RoutePredicateTypeEnum.EQUAL.name()); fallbackHeaderPredicate.setMatchValue(originalRouteName); fallbackHeaderPredicate.setCaseSensitive(true); - route.setHeaders(List.of(fallbackHeaderPredicate)); + if (route.getHeaders() == null) { + route.setHeaders(new ArrayList<>()); + } + route.getHeaders().add(fallbackHeaderPredicate); String fallbackStrategy = fallbackConfig.getFallbackStrategy(); List fallbackUpStreams; if (StringUtils.isEmpty(fallbackStrategy) || AiRouteFallbackStrategy.RANDOM.equals(fallbackStrategy)) { @@ -232,11 +264,59 @@ private void writeAuthConfigResources(String routeName, AiRouteAuthConfig authCo consumerService.updateAllowList(WasmPluginInstanceScope.ROUTE, routeName, allowedConsumers); } + private void writeModelRouteResources(AiModelPredicate modelPredicate) { + if (modelPredicate == null || !Boolean.TRUE.equals(modelPredicate.getEnabled())) { + return; + } + + final String pluginName = BuiltInPluginName.MODEL_ROUTER; + WasmPluginInstance instance = + wasmPluginInstanceService.query(WasmPluginInstanceScope.GLOBAL, null, pluginName, true); + if (instance == null) { + WasmPlugin plugin = wasmPluginService.query(pluginName, null); + if (plugin == null) { + throw new BusinessException("Plugin " + pluginName + " not found"); + } + instance = new WasmPluginInstance(); + instance.setPluginName(plugin.getName()); + instance.setPluginVersion(plugin.getPluginVersion()); + instance.setInternal(true); + instance.setScope(WasmPluginInstanceScope.GLOBAL); + } + instance.setEnabled(true); + + Map configurations = instance.getConfigurations(); + if (MapUtils.isEmpty(configurations)) { + // Just in case it is a readonly empty map. + configurations = new HashMap<>(); + instance.setConfigurations(configurations); + } + + configurations.put(MODEL_ROUTER_ENABLE_KEY, Boolean.TRUE); + configurations.put(ADD_HEADER_KEY_KEY, HigressConstants.MODEL_ROUTER_HEADER); + + wasmPluginInstanceService.addOrUpdate(instance); + } + private Route buildRoute(String routeName, AiRoute aiRoute) { Route route = new Route(); route.setName(routeName); route.setPath(new RoutePredicate(RoutePredicateTypeEnum.PRE.name(), "/", true)); route.setDomains(aiRoute.getDomains()); + + AiModelPredicate modelPredicate = aiRoute.getModelPredicate(); + if (modelPredicate != null && Boolean.TRUE.equals(modelPredicate.getEnabled()) + && StringUtils.isNotEmpty(modelPredicate.getPrefix())) { + KeyedRoutePredicate modelHeaderPredicate = new KeyedRoutePredicate(HigressConstants.MODEL_ROUTER_HEADER); + modelHeaderPredicate.setMatchType(RoutePredicateTypeEnum.EQUAL.name()); + modelHeaderPredicate.setMatchValue(modelPredicate.getPrefix()); + modelHeaderPredicate.setCaseSensitive(true); + if (route.getHeaders() == null) { + route.setHeaders(new ArrayList<>()); + } + route.getHeaders().add(modelHeaderPredicate); + } + return route; } @@ -257,20 +337,19 @@ private void setUpstreams(Route route, List upstreams) { } private void deleteAiRouteResources(String aiRouteName) { - String routeName = aiRouteName + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; - routeService.delete(routeName); + String resourceName = buildRouteResourceName(aiRouteName); + routeService.delete(resourceName); try { - kubernetesClientService.deleteEnvoyFilter(routeName); + kubernetesClientService.deleteEnvoyFilter(resourceName); } catch (ApiException e) { if (e.getCode() != HttpStatus.NOT_FOUND) { - throw new BusinessException("Error occurs when deleting the EnvoyFilter with name: " + routeName, e); + throw new BusinessException("Error occurs when deleting the EnvoyFilter with name: " + resourceName, e); } } - String fallbackRouteName = - aiRouteName + HigressConstants.FALLBACK_ROUTE_NAME_SUFFIX + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; - routeService.delete(fallbackRouteName); + String fallbackResourceName = buildFallbackRouteResourceName(aiRouteName); + routeService.delete(fallbackResourceName); } private void saveRoute(Route route) { @@ -282,4 +361,13 @@ private void saveRoute(Route route) { routeService.update(route); } } + + private static String buildRouteResourceName(String routeName) { + return CommonKey.AI_ROUTE_PREFIX + routeName + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + } + + private static String buildFallbackRouteResourceName(String routeName) { + return CommonKey.AI_ROUTE_PREFIX + routeName + HigressConstants.FALLBACK_ROUTE_NAME_SUFFIX + + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java index 3afd7845..e9eabcd0 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java @@ -57,6 +57,7 @@ public class LlmProviderServiceImpl implements LlmProviderService { static { PROVIDER_HANDLERS = Stream.of( new DefaultLlmProviderHandler(LlmProviderType.OPENAI, "api.openai.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.MOONSHOT, "api.moonshot.cn", 443, V1McpBridge.PROTOCOL_HTTPS), new DefaultLlmProviderHandler(LlmProviderType.QWEN, "dashscope.aliyuncs.com", 443, V1McpBridge.PROTOCOL_HTTPS)) .collect(Collectors.toMap(LlmProviderHandler::getType, p -> p)); diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java index 303c0b78..990825f1 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java @@ -21,6 +21,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.collections4.CollectionUtils; + import com.alibaba.higress.sdk.exception.BusinessException; import com.alibaba.higress.sdk.model.CommonPageQuery; import com.alibaba.higress.sdk.model.PaginatedResult; @@ -112,6 +114,13 @@ public void delete(String consumerName) { public void updateAllowList(WasmPluginInstanceScope scope, String target, List consumerNames) { for (CredentialHandler config : CREDENTIAL_HANDLERS.values()) { WasmPluginInstance instance = wasmPluginInstanceService.query(scope, target, config.getPluginName(), true); + if (CollectionUtils.isEmpty(consumerNames)) { + if (instance != null) { + wasmPluginInstanceService.delete(scope, target, config.getPluginName()); + } + continue; + } + if (instance == null) { instance = createInstance(scope, target, config.getPluginName()); } diff --git a/backend/sdk/src/main/resources/plugins/model-router/README.md b/backend/sdk/src/main/resources/plugins/model-router/README.md new file mode 100644 index 00000000..b63be35d --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-router/README.md @@ -0,0 +1,64 @@ +## 功能说明 +`model-router`插件实现了基于LLM协议中的model参数路由的功能 + +## 运行属性 + +插件执行阶段：`默认阶段` +插件执行优先级：`260` + +## 配置字段 + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | +| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | +| `enable` | bool | 选填 | false | 是否开启基于model参数路由 | +| `model_key` | string | 选填 | model | 请求body中model参数的位置 | +| `add_header_key` | string | 选填 | x-higress-llm-provider | 从model参数中解析出的provider名字放到哪个请求header中 | + + +## 效果说明 + +如下开启基于model参数路由的功能： + +```yaml +enable: true +``` + +开启后，插件将请求中 model 参数的 provider 部分（如果有）提取出来，设置到 x-higress-llm-provider 这个请求 header 中，用于后续路由，并将 model 参数重写为模型名称部分。举例来说，原生的 LLM 请求体是： + +```json +{ + "model": "qwen/qwen-long", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "higress项目主仓库的github地址是什么" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` + +经过这个插件后，将添加下面这个请求头(可以用于路由匹配)： + +x-higress-llm-provider: qwen + +原始的 LLM 请求体将被改成： + +```json +{ + "model": "qwen-long", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "higress项目主仓库的github地址是什么" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` diff --git a/backend/sdk/src/main/resources/plugins/model-router/README_EN.md b/backend/sdk/src/main/resources/plugins/model-router/README_EN.md new file mode 100644 index 00000000..4d2eaf1f --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-router/README_EN.md @@ -0,0 +1,63 @@ +## Function Description +The `model-router` plugin implements the functionality of routing based on the `model` parameter in the LLM protocol. + +## Runtime Properties + +Plugin Execution Phase: `Default Phase` +Plugin Execution Priority: `260` + +## Configuration Fields + +| Name | Data Type | Filling Requirement | Default Value | Description | +| -------------------- | ------------- | --------------------- | ---------------------- | ----------------------------------------------------- | +| `enable` | bool | Optional | false | Whether to enable routing based on the `model` parameter | +| `model_key` | string | Optional | model | The location of the `model` parameter in the request body | +| `add_header_key` | string | Optional | x-higress-llm-provider | The header where the parsed provider name from the `model` parameter will be placed | + +## Effect Description + +To enable routing based on the `model` parameter, use the following configuration: + +```yaml +enable: true +``` + +After enabling, the plugin extracts the provider part (if any) from the `model` parameter in the request, and sets it in the `x-higress-llm-provider` request header for subsequent routing. It also rewrites the `model` parameter to the model name part. For example, the original LLM request body is: + +```json +{ + "model": "openai/gpt-4o", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "What is the GitHub address for the main repository of the Higress project?" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` + +After processing by the plugin, the following request header (which can be used for routing matching) will be added: + +`x-higress-llm-provider: openai` + +The original LLM request body will be modified to: + +```json +{ + "model": "gpt-4o", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "What is the GitHub address for the main repository of the Higress project?" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` diff --git a/backend/sdk/src/main/resources/plugins/model-router/spec.yaml b/backend/sdk/src/main/resources/plugins/model-router/spec.yaml new file mode 100644 index 00000000..cf9c30c0 --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-router/spec.yaml @@ -0,0 +1,50 @@ +apiVersion: 1.0.0 +info: + gatewayMinVersion: "2.0.0" + category: ai + name: model-router + title: AI Model Router + x-title-i18n: + zh-CN: AI 模型路由 + description: Implements the functionality of routing based on the `model` parameter in the LLM protocol. + x-description-i18n: + zh-CN: 实现了基于 LLM 协议中的 model 参数路由的功能 + iconUrl: https://img.alicdn.com/imgextra/i1/O1CN018iKKih1iVx287RltL_!!6000000004419-2-tps-42-42.png + version: 1.0.0 + contact: + name: johnlanni +spec: + phase: default + priority: 260 + configSchema: + openAPIV3Schema: + type: object + properties: + enable: + type: boolean + title: Enabled + x-title-i18n: + zh-CN: 启用 + description: 是否开启基于 model 参数路由 + x-description-i18n: + zh-CN: 是否开启基于 model 参数路由 + model_key: + type: string + title: Model Key + x-title-i18n: + zh-CN: Model 字段名称 + description: The location of the `model` parameter in the request body + x-description-i18n: + zh-CN: 请求 body 中 model 参数的位置 + add_header_key: + type: string + title: Header Key + x-title-i18n: + zh-CN: Header 名称 + description: The header where the parsed provider name from the `model` parameter will be placed + x-description-i18n: + zh-CN: 从 model 参数中解析出的 provider 名字放到哪个请求 header 中 + example: + enable: true + model_key: model + add_header_key: x-higress-llm-provider diff --git a/backend/sdk/src/main/resources/plugins/plugins.properties b/backend/sdk/src/main/resources/plugins/plugins.properties index e15c9c3f..d6b90b46 100644 --- a/backend/sdk/src/main/resources/plugins/plugins.properties +++ b/backend/sdk/src/main/resources/plugins/plugins.properties @@ -26,6 +26,7 @@ ai-intent=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-intent:l ai-quota=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-quota:latest ai-agent=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-agent:latest ai-json-resp=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-json-resp:latest +model-router=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/model-router:latest # Auth basic-auth=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/basic-auth:latest