Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add tracing APIs and instrumentation #534

Merged
merged 3 commits into from
Sep 3, 2024
Merged

add tracing APIs and instrumentation #534

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4234b76
add tracing APIs and instrumentation
lucix-aws Aug 29, 2024
941c904
default tracer provider
lucix-aws Aug 29, 2024
99644da
map unrecognized k,v where possible
lucix-aws Sep 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public final class GoWriter extends SymbolWriter<GoWriter, ImportDeclarations> {

private static final Logger LOGGER = Logger.getLogger(GoWriter.class.getName());
private static final int DEFAULT_DOC_WRAP_LENGTH = 80;
private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]+)(:\\w)?");
private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]\\.+)(:\\w)?");
private final String fullPackageName;
private final boolean innerWriter;
private final List<String> buildTags = new ArrayList<>();
Expand Down Expand Up @@ -97,6 +97,11 @@ private void init() {
putFormatter('W', new GoWritableInjector());
putFormatter('D', new GoDependencyFormatter());

putContext("fmt.Sprintf", SmithyGoDependency.FMT.func("Sprintf"));
putContext("fmt.Errorf", SmithyGoDependency.FMT.func("Errorf"));
putContext("errors.As", SmithyGoDependency.ERRORS.func("As"));
putContext("context.Context", SmithyGoDependency.CONTEXT.func("Context"));

if (!innerWriter) {
packageDocs = new GoWriter(this.fullPackageName, true);
}
Expand Down
79 changes: 67 additions & 12 deletions ...n/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import software.amazon.smithy.aws.traits.ServiceTrait;
import software.amazon.smithy.codegen.core.SymbolProvider;
import software.amazon.smithy.go.codegen.auth.AuthSchemeResolverGenerator;
import software.amazon.smithy.go.codegen.auth.GetIdentityMiddlewareGenerator;
Expand Down Expand Up @@ -94,6 +95,7 @@ public void run() {
private GoWriter.Writable generate() {
return GoWriter.ChainWritable.of(
generateMetadata(),
generateObservabilityHelpers(),
generateClient(),
generateNew(),
generateGetOptions(),
Expand All @@ -103,6 +105,19 @@ private GoWriter.Writable generate() {
).compose();
}

private GoWriter.Writable generateObservabilityHelpers() {
return goTemplate("""
func operationTracer(p $tracerProvider:T) $tracer:T {
return p.Tracer($scope:S)
}
""",
Map.of(
"tracerProvider", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("TracerProvider"),
"tracer", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("Tracer"),
"scope", settings.getModuleName()
));
}

private GoWriter.Writable generateMetadata() {
var serviceId = settings.getService().toString();
for (var integration : integrations) {
Expand Down Expand Up @@ -324,7 +339,7 @@ func resolveAuthSchemes(options *Options) {
@SuppressWarnings("checkstyle:LineLength")
private GoWriter.Writable generateInvokeOperation() {
return goTemplate("""
func (c *Client) invokeOperation(ctx $context:T, opID string, params interface{}, optFns []func(*Options), stackFns ...func($stack:P, Options) error) (result interface{}, metadata $metadata:T, err error) {
func (c *Client) invokeOperation(ctx $context.Context:T, opID string, params interface{}, optFns []func(*Options), stackFns ...func($stack:P, Options) error) (result interface{}, metadata $metadata:T, err error) {
ctx = $clearStackValues:T(ctx)
$newStack:W
options := c.options.Copy()
Expand All @@ -348,25 +363,55 @@ private GoWriter.Writable generateInvokeOperation() {
}
}

$newStackHandler:W
result, metadata, err = handler.Handle(ctx, params)
tracer := operationTracer(options.TracerProvider)
spanName := $fmt.Sprintf:T("$tracingServiceId:L.%s", opID)

ctx = $withOperationTracer:T(ctx, tracer)

ctx, span := tracer.StartSpan(ctx, spanName, func (o $spanOptions:P) {
o.Kind = $spanKindClient:T
o.Properties.Set("rpc.system", "aws-api")
o.Properties.Set("rpc.method", opID)
o.Properties.Set("rpc.service", $tracingServiceId:S)
})
defer span.End()

handler := $newClientHandler:T(options.HTTPClient)
decorated := $decorateHandler:T(handler, stack)
result, metadata, err = decorated.Handle(ctx, params)
if err != nil {
span.SetProperty("error.go.type", $fmt.Sprintf:T("%T", err))
span.SetProperty("error.go.error", err.Error())

var aerr smithy.APIError
if $errors.As:T(err, &aerr) {
span.SetProperty("error.api.code", aerr.ErrorCode())
span.SetProperty("error.api.message", aerr.ErrorMessage())
span.SetProperty("error.api.fault", aerr.ErrorFault().String())
}

err = &$operationError:T{
ServiceID: ServiceID,
OperationName: opID,
Err: err,
}
}

span.SetProperty("error", err != nil)
if err == nil {
span.SetStatus($spanStatusOK:T)
} else {
span.SetStatus($spanStatusError:T)
}

return result, metadata, err
}
""",
MapUtils.of(
"context", GoStdlibTypes.Context.Context,
"stack", SmithyGoTypes.Middleware.Stack,
"metadata", SmithyGoTypes.Middleware.Metadata,
"clearStackValues", SmithyGoTypes.Middleware.ClearStackValues,
"newStack", generateNewStack(),
"newStackHandler", generateNewStackHandler(),
"operationError", SmithyGoTypes.Smithy.OperationError,
"resolvers", GoWriter.ChainWritable.of(
getConfigResolvers(
Expand All @@ -379,7 +424,17 @@ private GoWriter.Writable generateInvokeOperation() {
ConfigFieldResolver.Location.OPERATION,
ConfigFieldResolver.Target.FINALIZATION
).map(this::generateConfigFieldResolver).toList()
).compose()
).compose(),
"newClientHandler", SmithyGoDependency.SMITHY_HTTP_TRANSPORT.func("NewClientHandler"),
"decorateHandler", SmithyGoDependency.SMITHY_MIDDLEWARE.func("DecorateHandler")
),
Map.of(
"tracingServiceId", getTracingServiceId(),
"spanOptions", SmithyGoDependency.SMITHY_TRACING.struct("SpanOptions"),
"spanKindClient", SmithyGoDependency.SMITHY_TRACING.constSymbol("SpanKindClient"),
"withOperationTracer", SmithyGoDependency.SMITHY_TRACING.constSymbol("WithOperationTracer"),
"spanStatusOK", SmithyGoDependency.SMITHY_TRACING.constSymbol("SpanStatusOK"),
"spanStatusError", SmithyGoDependency.SMITHY_TRACING.constSymbol("SpanStatusError")
));
}

Expand All @@ -389,12 +444,6 @@ private GoWriter.Writable generateNewStack() {
SmithyGoTypes.Middleware.NewStack, SmithyGoTypes.Transport.Http.NewStackRequest);
}

private GoWriter.Writable generateNewStackHandler() {
ensureSupportedProtocol();
return goTemplate("handler := $T($T(options.HTTPClient), stack)",
SmithyGoTypes.Middleware.DecorateHandler, SmithyGoTypes.Transport.Http.NewClientHandler);
}

private void ensureSupportedProtocol() {
if (!applicationProtocol.isHttpProtocol()) {
throw new UnsupportedOperationException(
Expand Down Expand Up @@ -448,4 +497,10 @@ func addProtocolFinalizerMiddlewares(stack $P, options $L, operation string) err
SignRequestMiddlewareGenerator.generateAddToProtocolFinalizers()
).compose(false));
}

private String getTracingServiceId() {
return service.hasTrait(ServiceTrait.class)
? service.expectTrait(ServiceTrait.class).getSdkId().replaceAll("\\s", "")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting, do we really have services with spaces on the sdk id?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Many do -- here's a grep snippet. It's really almost the rule rather than the exception

iot-wireless.json                                                                                                                                                [13/1970]
16254:                    "sdkId": "IoT Wireless",                                                                                                                        
                                                                                                                                                                          
connect.json                                                                                                                                                              
1640:                    "sdkId": "Connect",                                                                                                                              
                                                                                                                                                                          
guardduty.json                                                                                                                                                            
6989:                    "sdkId": "GuardDuty",                                                                                                                            
                                                                                                                                                                          
lex-runtime-v2.json                                                                                                                                                       
57:                    "sdkId": "Lex Runtime V2",                                                                                                                         
                                                                                                                                                                          
route-53-domains.json                                                                                                                                                     
4974:                    "sdkId": "Route 53 Domains",                                                                                                                     

mediastore.json
1135:                    "sdkId": "MediaStore",

resource-explorer-2.json
1597:                    "sdkId": "Resource Explorer 2",

elasticache.json
453:                    "sdkId": "ElastiCache",

athena.json
275:                    "sdkId": "Athena",

migration-hub-refactor-spaces.json
3534:                    "sdkId": "Migration Hub Refactor Spaces",

kafkaconnect.json
1856:                    "sdkId": "KafkaConnect",

backup-gateway.json
128:                    "sdkId": "Backup Gateway",

: service.getId().getName();
}
}
1 change: 1 addition & 0 deletions ...smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public final class SmithyGoDependency {
public static final GoDependency SMITHY_AUTH_BEARER = smithy("auth/bearer");
public static final GoDependency SMITHY_ENDPOINTS = smithy("endpoints", "smithyendpoints");
public static final GoDependency SMITHY_ENDPOINT_RULESFN = smithy("endpoints/private/rulesfn");
public static final GoDependency SMITHY_TRACING = smithy("tracing");

public static final GoDependency GO_JMESPATH = goJmespath(null);
public static final GoDependency MATH = stdlib("math");
Expand Down
18 changes: 12 additions & 6 deletions .../src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import software.amazon.smithy.go.codegen.GoStdlibTypes;
import software.amazon.smithy.go.codegen.GoWriter;
import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
import software.amazon.smithy.go.codegen.SmithyGoDependency;
import software.amazon.smithy.go.codegen.SmithyGoTypes;
import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
import software.amazon.smithy.utils.MapUtils;
Expand Down Expand Up @@ -64,26 +65,31 @@ private GoWriter.Writable generateFields() {

private GoWriter.Writable generateBody() {
return goTemplate("""
rscheme := getResolvedAuthScheme(ctx)
innerCtx, span := $startSpan:T(ctx, "GetIdentity")
defer span.End()

rscheme := getResolvedAuthScheme(innerCtx)
if rscheme == nil {
return out, metadata, $errorf:T("no resolved auth scheme")
return out, metadata, $fmt.Errorf:T("no resolved auth scheme")
}

resolver := rscheme.Scheme.IdentityResolver(m.options)
if resolver == nil {
return out, metadata, $errorf:T("no identity resolver")
return out, metadata, $fmt.Errorf:T("no identity resolver")
}

identity, err := resolver.GetIdentity(ctx, rscheme.IdentityProperties)
identity, err := resolver.GetIdentity(innerCtx, rscheme.IdentityProperties)
if err != nil {
return out, metadata, $errorf:T("get identity: %w", err)
return out, metadata, $fmt.Errorf:T("get identity: %w", err)
}

ctx = setIdentity(ctx, identity)

span.End()
return next.HandleFinalize(ctx, in)
""",
MapUtils.of(
"errorf", GoStdlibTypes.Fmt.Errorf
"startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan")
));
}

Expand Down
10 changes: 9 additions & 1 deletion ...ain/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import software.amazon.smithy.go.codegen.GoStdlibTypes;
import software.amazon.smithy.go.codegen.GoWriter;
import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
import software.amazon.smithy.go.codegen.SmithyGoDependency;
import software.amazon.smithy.go.codegen.SmithyGoTypes;
import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
import software.amazon.smithy.utils.MapUtils;
Expand Down Expand Up @@ -66,6 +67,9 @@ private GoWriter.Writable generateFields() {

private GoWriter.Writable generateBody() {
return goTemplate("""
_, span := $3T(ctx, "ResolveAuthScheme")
defer span.End()

params := $1L(ctx, m.operation, getOperationInput(ctx), m.options)
options, err := m.options.AuthSchemeResolver.ResolveAuthSchemes(ctx, params)
if err != nil {
Expand All @@ -78,10 +82,14 @@ private GoWriter.Writable generateBody() {
}

ctx = setResolvedAuthScheme(ctx, scheme)

span.SetProperty("operation.auth.resolved_scheme_id", scheme.Scheme.SchemeID())
span.End()
return next.HandleFinalize(ctx, in)
""",
AuthParametersResolverGenerator.FUNC_NAME,
GoStdlibTypes.Fmt.Errorf
GoStdlibTypes.Fmt.Errorf,
SmithyGoDependency.SMITHY_TRACING.func("StartSpan")
);
}

Expand Down
24 changes: 15 additions & 9 deletions .../src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@
package software.amazon.smithy.go.codegen.auth;

import static software.amazon.smithy.go.codegen.GoStackStepMiddlewareGenerator.createFinalizeStepMiddleware;
import static software.amazon.smithy.go.codegen.GoWriter.emptyGoTemplate;
import static software.amazon.smithy.go.codegen.GoWriter.goTemplate;

import software.amazon.smithy.go.codegen.GoStdlibTypes;
import software.amazon.smithy.go.codegen.GoWriter;
import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
import software.amazon.smithy.go.codegen.SmithyGoDependency;
import software.amazon.smithy.go.codegen.SmithyGoTypes;
import software.amazon.smithy.go.codegen.endpoints.EndpointMiddlewareGenerator;
import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
Expand All @@ -39,7 +39,7 @@ public SignRequestMiddlewareGenerator(ProtocolGenerator.GenerationContext contex

public static GoWriter.Writable generateAddToProtocolFinalizers() {
return goTemplate("""
if err := stack.Finalize.Insert(&$L{}, $S, $T); err != nil {
if err := stack.Finalize.Insert(&$L{options: options}, $S, $T); err != nil {
return $T("add $L: %w", err)
}
""",
Expand All @@ -56,41 +56,47 @@ public GoWriter.Writable generate() {
}

private GoWriter.Writable generateFields() {
return emptyGoTemplate();
return goTemplate("""
options Options
""");
}

private GoWriter.Writable generateBody() {
return goTemplate("""
_, span := $startSpan:T(ctx, "SignRequest")
defer span.End()

req, ok := in.Request.($request:P)
if !ok {
return out, metadata, $errorf:T("unexpected transport type %T", in.Request)
return out, metadata, $fmt.Errorf:T("unexpected transport type %T", in.Request)
}

rscheme := getResolvedAuthScheme(ctx)
if rscheme == nil {
return out, metadata, $errorf:T("no resolved auth scheme")
return out, metadata, $fmt.Errorf:T("no resolved auth scheme")
}

identity := getIdentity(ctx)
if identity == nil {
return out, metadata, $errorf:T("no identity")
return out, metadata, $fmt.Errorf:T("no identity")
}

signer := rscheme.Scheme.Signer()
if signer == nil {
return out, metadata, $errorf:T("no signer")
return out, metadata, $fmt.Errorf:T("no signer")
}

if err := signer.SignRequest(ctx, req, identity, rscheme.SignerProperties); err != nil {
return out, metadata, $errorf:T("sign request: %w", err)
return out, metadata, $fmt.Errorf:T("sign request: %w", err)
}

span.End()
return next.HandleFinalize(ctx, in)
""",
MapUtils.of(
// FUTURE(#458) protocol generator should specify the transport type
"request", SmithyGoTypes.Transport.Http.Request,
"errorf", GoStdlibTypes.Fmt.Errorf
"startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan")
));
}
}
8 changes: 8 additions & 0 deletions ...rc/main/java/software/amazon/smithy/go/codegen/endpoints/EndpointMiddlewareGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import static software.amazon.smithy.go.codegen.GoStackStepMiddlewareGenerator.createFinalizeStepMiddleware;
import static software.amazon.smithy.go.codegen.GoWriter.goTemplate;
import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_TRACING;

import software.amazon.smithy.go.codegen.GoStdlibTypes;
import software.amazon.smithy.go.codegen.GoWriter;
Expand Down Expand Up @@ -72,6 +73,9 @@ private GoWriter.Writable generateBody() {
}

return goTemplate("""
_, span := $startSpan:T(ctx, "ResolveEndpoint")
defer span.End()

$pre:W

$assertRequest:W
Expand All @@ -84,9 +88,11 @@ private GoWriter.Writable generateBody() {

$post:W

span.End()
return next.HandleFinalize(ctx, in)
""",
MapUtils.of(
"startSpan", SMITHY_TRACING.func("StartSpan"),
"pre", generatePreResolutionHooks(),
"assertRequest", generateAssertRequest(),
"assertResolver", generateAssertResolver(),
Expand Down Expand Up @@ -132,6 +138,8 @@ private GoWriter.Writable generateResolveEndpoint() {
return out, metadata, $1T("failed to resolve service endpoint, %w", err)
}

span.SetProperty("operation.resolved_endpoint", endpt.URI.String())

if endpt.URI.RawPath == "" && req.URL.RawPath != "" {
endpt.URI.RawPath = endpt.URI.Path
}
Expand Down
Loading
Loading