feat(opentelemetry): add opentelemetry variables

apisix/cli/ngx_tpl.lua

@@ -635,6 +635,14 @@ http {
         proxy_ssl_trusted_certificate {* ssl.ssl_trusted_certificate *};
         {% end %}
 
+        # opentelemetry_set_ngx_var starts
+        {% if opentelemetry_set_ngx_var then %}
+        set $opentelemetry_context_traceparent          '';
+        set $opentelemetry_trace_id                     '';
+        set $opentelemetry_span_id                      '';
+        {% end %}
+        # opentelemetry_set_ngx_var ends
+
         # http server configuration snippet starts
         {% if http_server_configuration_snippet then %}
         {* http_server_configuration_snippet *}

apisix/cli/ops.lua

@@ -542,6 +542,10 @@ Please modify "admin_key" in conf/config.yaml .
         end
     end
 
+    local opentelemetry_set_ngx_var
+    if enabled_plugins["opentelemetry"] and yaml_conf.plugin_attr["opentelemetry"] then
+        opentelemetry_set_ngx_var = yaml_conf.plugin_attr["opentelemetry"].set_ngx_var
+    end
     -- Using template.render
     local sys_conf = {
         lua_path = env.pkg_path_org,
@@ -562,6 +566,7 @@ Please modify "admin_key" in conf/config.yaml .
         control_server_addr = control_server_addr,
         prometheus_server_addr = prometheus_server_addr,
         proxy_mirror_timeouts = proxy_mirror_timeouts,
+        opentelemetry_set_ngx_var = opentelemetry_set_ngx_var
     }
 
     if not yaml_conf.apisix then

apisix/plugins/opentelemetry.lua

@@ -47,6 +47,7 @@ local type    = type
 local pairs   = pairs
 local ipairs  = ipairs
 local unpack  = unpack
+local string_format = string.format
 
 local lrucache = core.lrucache.new({
     type = 'plugin', count = 128, ttl = 24 * 60 * 60,
@@ -112,6 +113,11 @@ local attr_schema = {
             },
             default = {},
         },
+        set_ngx_var = {
+          type = "boolean",
+          description = "set nginx variables",
+          default = false,
+        },
     },
 }
 
@@ -332,6 +338,17 @@ function _M.rewrite(conf, api_ctx)
         kind = span_kind.server,
         attributes = attributes,
     })
+
+    if plugin_info.set_ngx_var then
+      local span_context = ctx:span():context()
+      ngx_var.opentelemetry_context_traceparent = string_format("00-%s-%s-%02x",
+                                                                 span_context.trace_id,
+                                                                 span_context.span_id,
+                                                                 span_context.trace_flags)
+      ngx_var.opentelemetry_trace_id = span_context.trace_id
+      ngx_var.opentelemetry_span_id = span_context.span_id
+    end
+
     api_ctx.otel_context_token = ctx:attach()
 
     -- inject trace context into the headers of upstream HTTP request

conf/config-default.yaml

@@ -570,6 +570,7 @@ plugin_attr:          # Plugin attributes
       inactive_timeout: 1           # Set the timeout for spans to wait in the export queue before being sent,
                                     # if the queue is not full.
       max_export_batch_size: 16     # Set the maximum number of spans to include in each batch sent to the
+    set_ngx_var: false              # export opentelemetry variables to nginx variables
                                     # OpenTelemetry collector.
   prometheus:                               # Plugin: prometheus
     export_uri: /apisix/prometheus/metrics  # Set the URI for the Prometheus metrics endpoint.

docs/en/latest/plugins/opentelemetry.md

@@ -89,6 +89,29 @@ plugin_attr:
       max_export_batch_size: 2
 ```
 
+## Variables
+
+The following nginx variables are set by OpenTelemetry:
+
+- `opentelemetry_context_traceparent` -  [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), e.g.: `00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01`
+- `opentelemetry_trace_id` - Trace Id of the current span
+- `opentelemetry_span_id` -  Span Id of the current span
+
+How to use variables? you have to add it to your configuration file (`conf/config.yaml`):
+
+```yaml title="./conf/config.yaml"
+http:
+    enable_access_log: true
+    access_log: "/dev/stdout"
+    access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+    access_log_format_escape: json
+plugins:
+  - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
+```
+
 ## Enable Plugin
 
 To enable the Plugin, you have to add it to your configuration file (`conf/config.yaml`):

docs/zh/latest/plugins/opentelemetry.md

@@ -88,6 +88,29 @@ plugin_attr:
       max_export_batch_size: 2
 ```
 
+## 如何使用变量
+
+以下`nginx`变量是由`opentelemetry` 设置的。
+
+- `opentelemetry_context_traceparent` -  [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), 例如：`00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01`
+- `opentelemetry_trace_id` - 当前 span 的 trace_id
+- `opentelemetry_span_id` - 当前 span 的 span_id
+
+如何使用？你需要在配置文件（`./conf/config.yaml`）设置如下：
+
+```yaml title="./conf/config.yaml"
+http:
+    enable_access_log: true
+    access_log: "/dev/stdout"
+    access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+    access_log_format_escape: json
+plugins:
+  - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
+```
+
 ## 如何启用
 
 `opentelemetry` 插件默认为禁用状态，你需要在配置文件（`./conf/config.yaml`）中开启该插件：

t/cli/test_opentelemetry_set_ngx_var.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+echo '
+plugins:
+    - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
+' > conf/config.yaml
+
+make init
+
+if ! grep "set \$opentelemetry_context_traceparent          '';" conf/nginx.conf > /dev/null; then
+    echo "failed: opentelemetry_context_traceparent not found in nginx.conf"
+    exit 1
+fi
+
+if ! grep "set \$opentelemetry_trace_id                     '';" conf/nginx.conf > /dev/null; then
+    echo "failed: opentelemetry_trace_id not found in nginx.conf"
+    exit 1
+fi
+
+if ! grep "set \$opentelemetry_span_id                      '';" conf/nginx.conf > /dev/null; then
+    echo "failed: opentelemetry_span_id not found in nginx.conf"
+    exit 1
+fi
+
+
+echo "passed: opentelemetry_set_ngx_var configuration is validated"

t/plugin/opentelemetry3.t

@@ -0,0 +1,170 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+log_level("info");
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!$block->extra_yaml_config) {
+        my $extra_yaml_config = <<_EOC_;
+plugins:
+    - http-logger
+    - opentelemetry
+plugin_attr:
+    opentelemetry:
+        set_ngx_var: true
+        batch_span_processor:
+            max_export_batch_size: 1
+            inactive_timeout: 0.5
+_EOC_
+        $block->set_value("extra_yaml_config", $extra_yaml_config);
+    }
+
+    my $upstream_server_config = $block->upstream_server_config // <<_EOC_;
+    set \$opentelemetry_context_traceparent "";
+    set \$opentelemetry_trace_id "";
+    set \$opentelemetry_span_id "";
+    access_log logs/error.log opentelemetry_log;
+_EOC_
+
+    $block->set_value("upstream_server_config", $upstream_server_config);
+
+    my $http_config = $block->http_config // <<_EOC_;
+    log_format opentelemetry_log '{"time": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}';
+_EOC_
+
+    $block->set_value("http_config", $http_config);
+
+    if (!$block->extra_init_by_lua) {
+        my $extra_init_by_lua = <<_EOC_;
+-- mock exporter http client
+local client = require("opentelemetry.trace.exporter.http_client")
+client.do_request = function()
+    ngx.log(ngx.INFO, "opentelemetry export span")
+    return "ok"
+end
+_EOC_
+
+        $block->set_value("extra_init_by_lua", $extra_init_by_lua);
+    }
+
+
+    if (!$block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    $block;
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: add plugin metadata
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/http-logger',
+                ngx.HTTP_PUT,
+                [[{
+                    "log_format": {
+                        "opentelemetry_context_traceparent": "$opentelemetry_context_traceparent",
+                        "opentelemetry_trace_id": "$opentelemetry_trace_id",
+                        "opentelemetry_span_id": "$opentelemetry_span_id"
+                    }
+                }]]
+                )
+            if code >= 300 then
+                ngx.status = code
+                return body
+            end
+
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "http-logger": {
+                                "uri": "http://127.0.0.1:1980/log",
+                                "batch_max_size": 1,
+                                "max_retry_count": 1,
+                                "retry_delay": 2,
+                                "buffer_duration": 2,
+                                "inactive_timeout": 2,
+                                "concat_method": "new_line"
+                        },
+                        "opentelemetry": {
+                            "sampler": {
+                                "name": "always_on"
+                            }
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1982": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/hello"
+                }]]
+                )
+
+            if code >=300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 2: trigger opentelemetry with open set variables
+--- request
+GET /hello
+--- response_body
+hello world
+--- wait: 1
+--- grep_error_log eval
+qr/opentelemetry export span/
+--- grep_error_log_out
+opentelemetry export span
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/
+
+
+
+=== TEST 3: trigger opentelemetry with disable set variables
+--- yaml_config
+plugin_attr:
+    opentelemetry:
+        set_ngx_var: false
+--- request
+GET /hello
+--- response_body
+hello world
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"".*\}/