feat: ai-content-moderation plugin

apisix-master-0.rockspec

@@ -82,7 +82,7 @@ dependencies = {
     "lua-resty-t1k = 1.1.5",
     "brotli-ffi = 0.3-1",
     "lua-ffi-zlib = 0.6-0",
-    "api7-lua-resty-aws == 2.0.1-1",
+    "api7-lua-resty-aws == 2.0.2-1",
 }
 
 build = {

apisix/cli/config.lua

@@ -215,6 +215,7 @@ local _M = {
     "body-transformer",
     "ai-prompt-template",
     "ai-prompt-decorator",
+    "content-moderation",
     "proxy-mirror",
     "proxy-rewrite",
     "workflow",

apisix/plugins/content-moderation.lua

@@ -0,0 +1,155 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local aws = require("resty.aws")
+local aws_instance = aws()
+local http = require("resty.http")
+local next = next
+local pairs = pairs
+local unpack = unpack
+
+local aws_comprehend_schema = {
+    type = "object",
+    properties = {
+        access_key_id = { type = "string" },
+        secret_access_key = { type = "string" },
+        region = { type = "string" },
+        endpoint = {
+            type = "string",
+            pattern = [[^https?://]]
+        },
+    },
+    required = { "access_key_id", "secret_access_key", "region", }
+}
+
+local schema = {
+    type = "object",
+    properties = {
+        provider = {
+            type = "object",
+            properties = {
+                aws_comprehend = aws_comprehend_schema
+            },
+            -- change to oneOf/enum while implementing support for other services
+            required = { "aws_comprehend" }
+        },
+        moderation_categories = {
+            type = "object",
+            patternProperties = {
+                -- luacheck: push max code line length 300
+                ["^(PROFANITY|HATE_SPEECH|INSULT|HARASSMENT_OR_ABUSE|SEXUAL|VIOLENCE_OR_THREAT)$"] = {
+                -- luacheck: pop
+                type = "number",
+                    minimum = 0,
+                    maximum = 1
+                }
+            },
+            additionalProperties = false
+        },
+        toxicity_level = {
+            type = "number",
+            minimum = 0,
+            maximum = 1,
+            default = 0.5
+        },
+        reject_requests = {
+            type = "boolean",
+            default = true,
+        }
+    },
+    required = { "provider" },
+}
+
+
+local _M = {
+    version  = 0.1,
+    priority = 1040, -- TODO: might change
+    name     = "content-moderation",
+    schema   = schema,
+}
+
+
+function _M.check_schema(conf)
+    return core.schema.check(schema, conf)
+end
+
+function _M.rewrite(conf, ctx)
+    local body = core.request.get_body()
+    if not body then
+        return
+    end
+
+    local provider = conf.provider[next(conf.provider)]
+
+    -- TODO support secret
+    local credentials = aws_instance:Credentials({
+        accessKeyId = provider.access_key_id,
+        secretAccessKey = provider.secret_access_key,
+        sessionToken = provider.session_token,
+    })
+
+    local default_endpoint = "https://comprehend." .. provider.region .. ".amazonaws.com"
+    local scheme, host, port = unpack(http:parse_uri(provider.endpoint or default_endpoint))
+    local endpoint = scheme .. "://" .. host
+    aws_instance.config.endpoint = endpoint
+    aws_instance.config.ssl_verify = false
+
+    local comprehend = aws_instance:Comprehend({
+        credentials = credentials,
+        endpoint = endpoint,
+        region = provider.region,
+        port = port,
+    })
+
+    local res, err = comprehend:detectToxicContent({
+        LanguageCode = "en",
+        TextSegments = {
+            {
+                Text = body
+            }
+        },
+    })
+
+    if not res then
+        core.log.error("failed to send request to ", provider, ": ", err)
+        return 500, err
+    end
+
+    local result = res.body and res.body.ResultList and res.body.ResultList[1]
+    if not result then
+        return 500, "failed to get moderation result from response"
+    end
+
+
+    if conf.moderation_categories then
+        for _, item in pairs(result.Labels) do
+            if not conf.moderation_categories[item.Name] then
+                goto continue
+            end
+            if item.Score > conf.moderation_categories[item.Name] then
+                return 400, "request body exceeds " .. item.Name .. " threshold"
+            end
+            ::continue::
+        end
+    end
+
+    if result.Toxicity > conf.toxicity_level then
+        return 400, "request body exceeds toxicity threshold"
+    end
+end
+
+return _M

conf/config.yaml.example

@@ -478,6 +478,7 @@ plugins:                           # plugin list (sorted by priority)
   - body-transformer               # priority: 1080
   - ai-prompt-template             # priority: 1071
   - ai-prompt-decorator            # priority: 1070
+  - content-moderation             # priority: 1040 TODO: compare priority with other ai plugins
   - proxy-mirror                   # priority: 1010
   - proxy-rewrite                  # priority: 1008
   - workflow                       # priority: 1006

docs/en/latest/config.json

@@ -80,7 +80,8 @@
             "plugins/ext-plugin-post-req",
             "plugins/ext-plugin-post-resp",
             "plugins/inspect",
-            "plugins/ocsp-stapling"
+            "plugins/ocsp-stapling",
+            "plugins/content-moderation"
           ]
         },
         {

docs/en/latest/plugins/content-moderation.md

@@ -0,0 +1,199 @@
+---
+title: content-moderation
+keywords:
+  - Apache APISIX
+  - API Gateway
+  - Plugin
+  - content-moderation
+description: This document contains information about the Apache APISIX content-moderation Plugin.
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Description
+
+The `content-moderation` plugin processes the request body to check for toxicity and rejects the request if it exceeds the configured threshold.
+
+## Plugin Attributes
+
+| **Field**                                 | **Required** | **Type** | **Description**                                                                                                                          |
+| ----------------------------------------- | ------------ | -------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
+| provider.aws_comprehend.access_key_id     | Yes          | String   | AWS access key ID                                                                                                                        |
+| provider.aws_comprehend.secret_access_key | Yes          | String   | AWS secret access key                                                                                                                    |
+| provider.aws_comprehend.region            | Yes          | String   | AWS region                                                                                                                               |
+| provider.aws_comprehend.endpoint          | No           | String   | AWS Comprehend service endpoint. Must match the pattern `^https?://`                                                                     |
+| moderation_categories                     | No           | Object   | Configuration for moderation categories. Must be one of: PROFANITY, HATE_SPEECH, INSULT, HARASSMENT_OR_ABUSE, SEXUAL, VIOLENCE_OR_THREAT |
+| toxicity_level                            | No           | Number   | Threshold for overall toxicity detection. Range: 0 - 1. Default: 0.5                                                                     |
+
+## Example usage
+
+Create a route with the `content-moderation` plugin like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/1" -X PUT \
+  -H "X-API-KEY: ${ADMIN_API_KEY}" \
+  -d '{
+    "uri": "/post",
+    "plugins": {
+      "content-moderation": {
+        "provider": {
+          "aws_comprehend": {
+            "access_key_id": "access",
+            "secret_access_key": "ea+secret",
+            "region": "us-east-1"
+          }
+        },
+        "moderation_categories": {
+          "PROFANITY": 0.5
+        }
+      }
+    },
+    "upstream": {
+      "type": "roundrobin",
+      "nodes": {
+        "httpbin.org:80": 1
+      }
+    }
+  }'
+```
+
+Now send a request:
+
+```shell
+curl http://127.0.0.1:9080/post -i -XPOST  -H 'Content-Type: application/json' -d '{
+  "info": "<some very seriously profane message>"
+}'
+```
+
+Then the request will be blocked with error like this:
+
+```text
+HTTP/1.1 400 Bad Request
+Date: Fri, 30 Aug 2024 11:21:21 GMT
+Content-Type: text/plain; charset=utf-8
+Transfer-Encoding: chunked
+Connection: keep-alive
+Server: APISIX/3.10.0
+
+request body exceeds toxicity threshold
+```
+
+Send a request with normal request body:
+
+```shell
+curl http://127.0.0.1:9080/post -i -XPOST  -H 'Content-Type: application/json' -d '{
+  "info": "APISIX is wonderful"
+}'
+```
+
+This request will be proxied normally to the upstream.
+
+```text
+HTTP/1.1 200 OK
+Content-Type: application/json
+Content-Length: 530
+Connection: keep-alive
+Date: Fri, 30 Aug 2024 11:21:55 GMT
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Credentials: true
+Server: APISIX/3.10.0
+
+{
+  "args": {},
+  "data": "",
+  "files": {},
+  "form": {
+    "do you know what is alpha murder method? I will teach you with care": ""
+  },
+  "headers": {
+    "Accept": "*/*",
+    "Content-Length": "67",
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Host": "127.0.0.1",
+    "User-Agent": "curl/8.7.1",
+    "X-Amzn-Trace-Id": "Root=1-66d1ab53-0860444b1b01a3f93c7003f4",
+    "X-Forwarded-Host": "127.0.0.1"
+  },
+  "json": null,
+  "origin": "127.0.0.1, 163.53.25.129",
+  "url": "http://127.0.0.1/post"
+}
+```
+
+You can also configure filters on other moderation categories like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/1" -X PUT \
+  -H "X-API-KEY: ${ADMIN_API_KEY}" \
+  -d '{
+    "uri": "/post",
+    "plugins": {
+      "content-moderation": {
+        "provider": {
+          "aws_comprehend": {
+            "access_key_id": "access",
+            "secret_access_key": "ea+secret",
+            "region": "us-east-1"
+          }
+        },
+        "moderation_categories": {
+          "PROFANITY": 0.5,
+          "HARASSMENT_OR_ABUSE": 0.7,
+          "SEXUAL": 0.2
+        }
+      }
+    },
+    "upstream": {
+      "type": "roundrobin",
+      "nodes": {
+        "httpbin.org:80": 1
+      }
+    }
+  }'
+```
+
+If none of the `moderation_categories` are configured, request bodies will be moderated on the basis of overall toxicity.
+The default `toxicity_level` is 0.5, it can be configured like so.
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/1" -X PUT \
+  -H "X-API-KEY: ${ADMIN_API_KEY}" \
+  -d '{
+    "uri": "/post",
+    "plugins": {
+      "content-moderation": {
+        "provider": {
+          "aws_comprehend": {
+            "access_key_id": "access",
+            "secret_access_key": "ea+secret",
+            "region": "us-east-1"
+          }
+        }
+        "toxicity_level": 0.7
+      }
+    },
+    "upstream": {
+      "type": "roundrobin",
+      "nodes": {
+        "httpbin.org:80": 1
+      }
+    }
+  }'
+```

t/admin/plugins.t

@@ -95,6 +95,7 @@ proxy-cache
 body-transformer
 ai-prompt-template
 ai-prompt-decorator
+content-moderation
 proxy-mirror
 proxy-rewrite
 workflow