Skip to content

Commit

Permalink
compression support for HTTP exporter
Browse files Browse the repository at this point in the history
  • Loading branch information
mamunto committed Nov 6, 2024
1 parent d0006d3 commit 784a4e6
Show file tree
Hide file tree
Showing 8 changed files with 206 additions and 23 deletions.
10 changes: 8 additions & 2 deletions Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ let package = Package(
.library(name: "DatadogExporter", type: .static, targets: ["DatadogExporter"]),
.library(name: "NetworkStatus", type: .static, targets: ["NetworkStatus"]),
.library(name: "OTelSwiftLog", type: .static, targets: ["OTelSwiftLog"]),
.library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
.executable(name: "simpleExporter", targets: ["SimpleExporter"]),
.executable(name: "OTLPExporter", targets: ["OTLPExporter"]),
.executable(name: "OTLPHTTPExporter", targets: ["OTLPHTTPExporter"]),
Expand Down Expand Up @@ -110,14 +111,18 @@ let package = Package(
"OpenTelemetryProtocolExporterCommon",
.product(name: "GRPC", package: "grpc-swift")],
path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
.target(name: "DataCompression",
dependencies: [],
path: "Sources/Exporters/DataCompression"),
.target(name: "StdoutExporter",
dependencies: ["OpenTelemetrySdk"],
path: "Sources/Exporters/Stdout"),
.target(name: "InMemoryExporter",
dependencies: ["OpenTelemetrySdk"],
path: "Sources/Exporters/InMemory"),
.target(name: "DatadogExporter",
dependencies: ["OpenTelemetrySdk"],
dependencies: ["OpenTelemetrySdk",
"DataCompression"],
path: "Sources/Exporters/DatadogExporter",
exclude: ["NOTICE", "README.md"]),
.target(name: "PersistenceExporter",
Expand Down Expand Up @@ -163,6 +168,7 @@ let package = Package(
.testTarget(name: "OpenTelemetryProtocolExporterTests",
dependencies: ["OpenTelemetryProtocolExporterGrpc",
"OpenTelemetryProtocolExporterHttp",
"DataCompression",
.product(name: "NIO", package: "swift-nio"),
.product(name: "NIOHTTP1", package: "swift-nio"),
.product(name: "NIOTestUtils", package: "swift-nio")],
Expand Down Expand Up @@ -190,7 +196,7 @@ let package = Package(
path: "Examples/OTLP Exporter",
exclude: ["README.md"]),
.target(name: "OTLPHTTPExporter",
dependencies: ["OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
dependencies: ["OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
path: "Examples/OTLP HTTP Exporter",
exclude: ["README.md"]),
.target(name: "PrometheusSample",
Expand Down
10 changes: 8 additions & 2 deletions Package@swift-5.6.swift
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ let package = Package(
.library(name: "DatadogExporter", type: .static, targets: ["DatadogExporter"]),
.library(name: "NetworkStatus", type: .static, targets: ["NetworkStatus"]),
.library(name: "OTelSwiftLog", type: .static, targets: ["OTelSwiftLog"]),
.library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
.executable(name: "simpleExporter", targets: ["SimpleExporter"]),
.executable(name: "OTLPExporter", targets: ["OTLPExporter"]),
.executable(name: "OTLPHTTPExporter", targets: ["OTLPHTTPExporter"]),
Expand Down Expand Up @@ -115,14 +116,18 @@ let package = Package(
"OpenTelemetryProtocolExporterCommon",
.product(name: "GRPC", package: "grpc-swift")],
path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
.target(name: "DataCompression",
dependencies: [],
path: "Sources/Exporters/DataCompression"),
.target(name: "StdoutExporter",
dependencies: ["OpenTelemetrySdk"],
path: "Sources/Exporters/Stdout"),
.target(name: "InMemoryExporter",
dependencies: ["OpenTelemetrySdk"],
path: "Sources/Exporters/InMemory"),
.target(name: "DatadogExporter",
dependencies: ["OpenTelemetrySdk"],
dependencies: ["OpenTelemetrySdk",
"DataCompression"],
path: "Sources/Exporters/DatadogExporter",
exclude: ["NOTICE", "README.md"]),
.target(name: "PersistenceExporter",
Expand Down Expand Up @@ -173,6 +178,7 @@ let package = Package(
.testTarget(name: "OpenTelemetryProtocolExporterTests",
dependencies: ["OpenTelemetryProtocolExporterGrpc",
"OpenTelemetryProtocolExporterHttp",
"DataCompression",
.product(name: "NIO", package: "swift-nio"),
.product(name: "NIOHTTP1", package: "swift-nio"),
.product(name: "NIOTestUtils", package: "swift-nio")],
Expand Down Expand Up @@ -207,7 +213,7 @@ let package = Package(
),
.executableTarget(
name: "OTLPHTTPExporter",
dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
path: "Examples/OTLP HTTP Exporter",
exclude: ["README.md"]
),
Expand Down
10 changes: 8 additions & 2 deletions Package@swift-5.9.swift
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ let package = Package(
.library(name: "PersistenceExporter", targets: ["PersistenceExporter"]),
.library(name: "InMemoryExporter", targets: ["InMemoryExporter"]),
.library(name: "OTelSwiftLog", targets: ["OTelSwiftLog"]),
.library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
.executable(name: "ConcurrencyContext", targets: ["ConcurrencyContext"]),
.executable(name: "loggingTracer", targets: ["LoggingTracer"]),
],
Expand Down Expand Up @@ -70,6 +71,9 @@ let package = Package(
"OpenTelemetryProtocolExporterCommon",
.product(name: "GRPC", package: "grpc-swift")],
path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
.target(name: "DataCompression",
dependencies: [],
path: "Sources/Exporters/DataCompression"),
.target(name: "StdoutExporter",
dependencies: ["OpenTelemetrySdk"],
path: "Sources/Exporters/Stdout"),
Expand Down Expand Up @@ -100,6 +104,7 @@ let package = Package(
.testTarget(name: "OpenTelemetryProtocolExporterTests",
dependencies: ["OpenTelemetryProtocolExporterGrpc",
"OpenTelemetryProtocolExporterHttp",
"DataCompression",
.product(name: "NIO", package: "swift-nio"),
.product(name: "NIOHTTP1", package: "swift-nio"),
.product(name: "NIOTestUtils", package: "swift-nio")],
Expand Down Expand Up @@ -240,7 +245,7 @@ extension Package {
),
.executableTarget(
name: "OTLPHTTPExporter",
dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
path: "Examples/OTLP HTTP Exporter",
exclude: ["README.md"]
),
Expand All @@ -256,7 +261,8 @@ extension Package {
dependencies: ["ResourceExtension", "OpenTelemetrySdk"],
path: "Tests/InstrumentationTests/SDKResourceExtensionTests"),
.target(name: "DatadogExporter",
dependencies: ["OpenTelemetrySdk"],
dependencies: ["OpenTelemetrySdk",
"DataCompression"],
path: "Sources/Exporters/DatadogExporter",
exclude: ["NOTICE", "README.md"]),
.testTarget(name: "DatadogExporterTests",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import Foundation
import Compression

extension Data
public extension Data
{
/// Compresses the data.
/// - parameter withAlgorithm: Compression algorithm to use. See the `CompressionAlgorithm` type
Expand Down Expand Up @@ -247,7 +247,7 @@ extension Data

/// Calculate the Adler32 checksum of the data.
/// - returns: Adler32 checksum type. Can still be further advanced.
func adler32() -> Adler32
internal func adler32() -> Adler32
{
var res = Adler32()
res.advance(withChunk: self)
Expand All @@ -256,7 +256,7 @@ extension Data

/// Calculate the Crc32 checksum of the data.
/// - returns: Crc32 checksum type. Can still be further advanced.
func crc32() -> Crc32
internal func crc32() -> Crc32
{
var res = Crc32()
res.advance(withChunk: self)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

import DataCompression
import Foundation

/// Builds `URLRequest` for sending data to Datadog.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@

import Foundation

public enum CompressionType {
case gzip
case deflate
case none
}

public struct OtlpConfiguration {
public static let DefaultTimeoutInterval : TimeInterval = TimeInterval(10)

Expand All @@ -23,9 +29,15 @@ public struct OtlpConfiguration {
// let compression
public let headers : [(String,String)]?
public let timeout : TimeInterval
public let compression: CompressionType

public init(timeout : TimeInterval = OtlpConfiguration.DefaultTimeoutInterval, headers: [(String,String)]? = nil) {
public init(
timeout : TimeInterval = OtlpConfiguration.DefaultTimeoutInterval,
compression: CompressionType = .gzip,
headers: [(String,String)]? = nil
) {
self.headers = headers
self.timeout = timeout
self.compression = compression
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// SPDX-License-Identifier: Apache-2.0
//

import DataCompression
import Foundation
import SwiftProtobuf
import OpenTelemetryProtocolExporterCommon
Expand All @@ -28,20 +29,41 @@ public class OtlpHttpExporterBase {
}
}

public func createRequest(body: Message, endpoint: URL) -> URLRequest {
var request = URLRequest(url: endpoint)

do {
request.httpMethod = "POST"
request.httpBody = try body.serializedData()
request.setValue(Headers.getUserAgentHeader(), forHTTPHeaderField: Constants.HTTP.userAgent)
request.setValue("application/x-protobuf", forHTTPHeaderField: "Content-Type")
} catch {
print("Error serializing body: \(error)")
public func createRequest(body: Message, endpoint: URL) -> URLRequest {
var request = URLRequest(url: endpoint)

do {
let rawData = try body.serializedData()
request.httpMethod = "POST"
request.setValue(Headers.getUserAgentHeader(), forHTTPHeaderField: Constants.HTTP.userAgent)
request.setValue("application/x-protobuf", forHTTPHeaderField: "Content-Type")

var compressedData = rawData
switch config.compression {
case .gzip:
if let data = rawData.gzip() {
compressedData = data
request.setValue("gzip", forHTTPHeaderField: "Content-Encoding")
}

case .deflate:
if let data = rawData.deflate() {
compressedData = data
request.setValue("deflate", forHTTPHeaderField: "Content-Encoding")
}

case .none:
break
}
// Apply final data. Could be compressed or raw
// but it doesn't matter here
request.httpBody = compressedData
} catch {
print("Error serializing body: \(error)")
}

return request
}

return request
}

public func shutdown(explicitTimeout: TimeInterval? = nil) {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0
//

import Foundation
#if canImport(FoundationNetworking)
import FoundationNetworking
#endif
import Logging
import NIO
import NIOHTTP1
import NIOTestUtils
import OpenTelemetryApi
import OpenTelemetryProtocolExporterCommon
import DataCompression
@testable import OpenTelemetryProtocolExporterHttp
@testable import OpenTelemetrySdk
import XCTest

class OtlpHttpExporterBaseTests: XCTestCase {

var exporter: OtlpHttpExporterBase!
var spans: [SpanData] = []

override func setUp() {
super.setUp()

spans = []
let endpointName1 = "/api/foo" + String(Int.random(in: 1...100))
let endpointName2 = "/api/bar" + String(Int.random(in: 100...500))
spans.append(generateFakeSpan(endpointName: endpointName1))
spans.append(generateFakeSpan(endpointName: endpointName2))
}

// Test for .gzip compression
func testCreateRequestWithGzipCompression() {
let config = OtlpConfiguration(compression: .gzip)

exporter = OtlpHttpExporterBase(
endpoint: URL(string: "http://example.com")!,
config: config
)

let body = Opentelemetry_Proto_Collector_Trace_V1_ExportTraceServiceRequest.with {
$0.resourceSpans = SpanAdapter.toProtoResourceSpans(spanDataList: spans)
}

let request = exporter.createRequest(body: body, endpoint: URL(string: "http://example.com")!)

/// gzip
let data = try! body.serializedData().gzip()

// Verify Content-Encoding header is set to "gzip"
XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Encoding"), "gzip")
XCTAssertNotNil(request.httpBody)
XCTAssertEqual(request.httpBody!.count, data!.count)
}

// Test for .deflate compression
func testCreateRequestWithDeflateCompression() {
let config = OtlpConfiguration(compression: .deflate)

exporter = OtlpHttpExporterBase(
endpoint: URL(string: "http://example.com")!,
config: config
)

let body = Opentelemetry_Proto_Collector_Trace_V1_ExportTraceServiceRequest.with {
$0.resourceSpans = SpanAdapter.toProtoResourceSpans(spanDataList: spans)
}

let request = exporter.createRequest(body: body, endpoint: URL(string: "http://example.com")!)

/// deflate
let data = try! body.serializedData().deflate()

// Verify Content-Encoding header is set to "deflate"
XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Encoding"), "deflate")
XCTAssertNotNil(request.httpBody)
XCTAssertEqual(request.httpBody!.count, data!.count)
}

// Test for .none compression (no compression)
func testCreateRequestWithNoCompression() {
let config = OtlpConfiguration(compression: .none)

exporter = OtlpHttpExporterBase(
endpoint: URL(string: "http://example.com")!,
config: config
)

let body = Opentelemetry_Proto_Collector_Trace_V1_ExportTraceServiceRequest.with {
$0.resourceSpans = SpanAdapter.toProtoResourceSpans(spanDataList: spans)
}

let request = exporter.createRequest(body: body, endpoint: URL(string: "http://example.com")!)

let data = try! body.serializedData()

// Verify Content-Encoding header is set to "deflate"
XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Encoding"), nil)
XCTAssertNotNil(request.httpBody)
XCTAssertEqual(request.httpBody!.count, data.count)
}

private func generateFakeSpan(endpointName: String = "/api/endpoint") -> SpanData {
let duration = 0.9
let start = Date()
let end = start.addingTimeInterval(duration)
let testattributes: [String: AttributeValue] = ["foo": AttributeValue("bar")!, "fizz": AttributeValue("buzz")!]

var testData = SpanData(traceId: TraceId.random(),
spanId: SpanId.random(),
name: "GET " + endpointName,
kind: SpanKind.server,
startTime: start,
endTime: end,
totalAttributeCount: 2)
testData.settingAttributes(testattributes)
testData.settingTotalAttributeCount(2)
testData.settingHasEnded(true)
testData.settingTotalRecordedEvents(0)
testData.settingLinks([SpanData.Link]())
testData.settingTotalRecordedLinks(0)
testData.settingStatus(.ok)

return testData
}
}

0 comments on commit 784a4e6

Please sign in to comment.