Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(minipipeline): adjust how we compute TCP and TLS anomalies #1410

Merged
merged 2 commits into from
Nov 30, 2023
Merged

refactor(minipipeline): adjust how we compute TCP and TLS anomalies #1410

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3988216
refactor(minipipeline): adjust how we compute TLS anomalies
bassosimone Nov 30, 2023
3416367
[ci skip] Update internal/minipipeline/analysis.go
bassosimone Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions internal/cmd/minipipeline/testdata/analysis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": {},
"HTTPDiffBodyProportionFactor": 1,
Expand All @@ -17,8 +19,6 @@
"HTTPFinalResponsesWithTLS": {
"4": true
},
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": {},
"TCPTransactionsWithUnexplainedUnexpectedFailures": {}
}
4 changes: 2 additions & 2 deletions internal/cmd/minipipeline/testdata/analysis_classic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": null,
"HTTPDiffBodyProportionFactor": 1,
Expand All @@ -17,8 +19,6 @@
"HTTPFinalResponsesWithTLS": {
"4": true
},
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": {},
"TCPTransactionsWithUnexplainedUnexpectedFailures": {}
}
167 changes: 84 additions & 83 deletions internal/minipipeline/analysis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ func AnalyzeWebObservations(container *WebObservationsContainer) *WebAnalysis {
analysis.ComputeDNSLookupSuccessWithInvalidAddressesClassic(container)
analysis.ComputeDNSLookupUnexpectedFailure(container)

analysis.ComputeTCPConnectUnexpectedFailure(container)
analysis.ComputeTLSHandshakeUnexpectedFailure(container)

analysis.ComputeDNSExperimentFailure(container)
analysis.ComputeDNSPossiblyNonexistingDomains(container)

analysis.ComputeTCPTransactionsWithUnexpectedTCPConnectFailures(container)
analysis.ComputeTCPTransactionsWithUnexpectedTLSHandshakeFailures(container)
analysis.ComputeTCPTransactionsWithUnexpectedHTTPFailures(container)
analysis.ComputeTCPTransactionsWithUnexplainedUnexpectedFailures(container)

Expand Down Expand Up @@ -47,6 +48,12 @@ type WebAnalysis struct {
// DNSLookupUnexpectedFailure contains DNS transactions with unexpected failures.
DNSLookupUnexpectedFailure Set[int64]

// TCPConnectUnexpectedFailure contains TCP endpoint transactions with unexpected failures.
TCPConnectUnexpectedFailure Set[int64]

// TLSHandshakeUnexpectedFailure contains TLS endpoint transactions with unexpected failures.
TLSHandshakeUnexpectedFailure Set[int64]

// DNSExperimentFailure is the first failure experienced by a getaddrinfo-like resolver.
DNSExperimentFailure optional.Value[string]

Expand Down Expand Up @@ -86,14 +93,6 @@ type WebAnalysis struct {
// itself with whether there's control data, because TLS suffices.
HTTPFinalResponsesWithTLS optional.Value[map[int64]bool]

// TCPTransactionsWithUnexpectedTCPConnectFailures contains the TCP transaction IDs that
// contain TCP connect failures while the control measurement succeeded.
TCPTransactionsWithUnexpectedTCPConnectFailures optional.Value[map[int64]bool]

// TCPTransactionsWithUnexpectedTLSHandshakeFailures contains the TCP transaction IDs that
// contain TLS handshake failures while the control measurement succeeded.
TCPTransactionsWithUnexpectedTLSHandshakeFailures optional.Value[map[int64]bool]

// TCPSTransactionsWithUnexpectedHTTPFailures contains the TCP transaction IDs that
// contain HTTP failures while the control measurement succeeded.
TCPTransactionsWithUnexpectedHTTPFailures optional.Value[map[int64]bool]
Expand Down Expand Up @@ -262,6 +261,81 @@ func (wa *WebAnalysis) ComputeDNSLookupUnexpectedFailure(c *WebObservationsConta
}
}

// ComputeTCPConnectUnexpectedFailure computes the TCPConnectUnexpectedFailure field.
func (wa *WebAnalysis) ComputeTCPConnectUnexpectedFailure(c *WebObservationsContainer) {
for _, obs := range c.KnownTCPEndpoints {
// dials once we started following redirects should not be considered
if obs.TagDepth.IsNone() || obs.TagDepth.Unwrap() != 0 {
continue
}

// handle the case where there is no measurement
if obs.TCPConnectFailure.IsNone() {
continue
}

// handle the case where there is no control information
if obs.ControlTCPConnectFailure.IsNone() {
continue
}

// handle the case where both the probe and the control fail
if obs.TCPConnectFailure.Unwrap() != "" && obs.ControlTCPConnectFailure.Unwrap() != "" {
continue
}

// handle the case where only the control fails
if obs.ControlTCPConnectFailure.Unwrap() != "" {
continue
}

// handle the case where only the probe fails
if obs.TCPConnectFailure.Unwrap() != "" {
if utilsTCPConnectFailureSeemsMisconfiguredIPv6(obs) {
continue
}
wa.TCPConnectUnexpectedFailure.Add(obs.EndpointTransactionID.Unwrap())
continue
}
}
}

// ComputeTLSHandshakeUnexpectedFailure computes the TLSHandshakeUnexpectedFailure field.
func (wa *WebAnalysis) ComputeTLSHandshakeUnexpectedFailure(c *WebObservationsContainer) {
for _, obs := range c.KnownTCPEndpoints {
// handshakes once we started following redirects should not be considered
if obs.TagDepth.IsNone() || obs.TagDepth.Unwrap() != 0 {
continue
}

// handle the case where there is no measurement
if obs.TLSHandshakeFailure.IsNone() {
continue
}

// handle the case where there is no control information
if obs.ControlTLSHandshakeFailure.IsNone() {
continue
}

// handle the case where both the probe and the control fail
if obs.TLSHandshakeFailure.Unwrap() != "" && obs.ControlTCPConnectFailure.Unwrap() != "" {
continue
}

// handle the case where only the control fails
if obs.ControlTLSHandshakeFailure.Unwrap() != "" {
continue
}

// handle the case where only the probe fails
if obs.TLSHandshakeFailure.Unwrap() != "" {
wa.TLSHandshakeUnexpectedFailure.Add(obs.EndpointTransactionID.Unwrap())
continue
}
}
}

// ComputeDNSExperimentFailure computes the DNSExperimentFailure field.
func (wa *WebAnalysis) ComputeDNSExperimentFailure(c *WebObservationsContainer) {

Expand Down Expand Up @@ -366,79 +440,6 @@ func (wa *WebAnalysis) ComputeDNSPossiblyNonexistingDomains(c *WebObservationsCo
wa.DNSPossiblyNonexistingDomains = optional.Some(state)
}

// ComputeTCPTransactionsWithUnexpectedTCPConnectFailures computes the TCPTransactionsWithUnexpectedTCPConnectFailures field.
func (wa *WebAnalysis) ComputeTCPTransactionsWithUnexpectedTCPConnectFailures(c *WebObservationsContainer) {
var state map[int64]bool

for _, obs := range c.KnownTCPEndpoints {
// we cannot do anything unless we have both records
if obs.TCPConnectFailure.IsNone() || obs.ControlTCPConnectFailure.IsNone() {
continue
}

// flip state from None to empty once we have seen the first
// suitable set of measurement/control pairs
if state == nil {
state = make(map[int64]bool)
}

// skip cases with no failures
if obs.TCPConnectFailure.Unwrap() == "" {
continue
}

// skip cases where also the control failed
if obs.ControlTCPConnectFailure.Unwrap() != "" {
continue
}

// skip cases where the root cause could be a misconfigured IPv6 stack
if utilsTCPConnectFailureSeemsMisconfiguredIPv6(obs) {
continue
}

// update state
state[obs.EndpointTransactionID.Unwrap()] = true
}

// note that optional.Some constructs None if state is nil
wa.TCPTransactionsWithUnexpectedTCPConnectFailures = optional.Some(state)
}

// ComputeTCPTransactionsWithUnexpectedTLSHandshakeFailures computes the TCPTransactionsWithUnexpectedTLSHandshakeFailures field.
func (wa *WebAnalysis) ComputeTCPTransactionsWithUnexpectedTLSHandshakeFailures(c *WebObservationsContainer) {
var state map[int64]bool

for _, obs := range c.KnownTCPEndpoints {
// we cannot do anything unless we have both records
if obs.TLSHandshakeFailure.IsNone() || obs.ControlTLSHandshakeFailure.IsNone() {
continue
}

// flip state from None to empty once we have seen the first
// suitable set of measurement/control pairs
if state == nil {
state = make(map[int64]bool)
}

// skip cases with no failures
if obs.TLSHandshakeFailure.Unwrap() == "" {
continue
}

// skip cases where also the control failed
if obs.ControlTLSHandshakeFailure.Unwrap() != "" {
continue
}

// update state
state[obs.EndpointTransactionID.Unwrap()] = true
}

// note that optional.Some constructs None if state is nil
wa.TCPTransactionsWithUnexpectedTLSHandshakeFailures = optional.Some(state)
}

// ComputeTCPTransactionsWithUnexpectedHTTPFailures computes the TCPTransactionsWithUnexpectedHTTPFailures field.
func (wa *WebAnalysis) ComputeTCPTransactionsWithUnexpectedHTTPFailures(c *WebObservationsContainer) {
var state map[int64]bool
Expand Down
4 changes: 2 additions & 2 deletions ...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": {},
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
4 changes: 2 additions & 2 deletions ...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": null,
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
4 changes: 2 additions & 2 deletions ...tdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": {},
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
4 changes: 2 additions & 2 deletions ...bconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/analysis_classic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": null,
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
4 changes: 2 additions & 2 deletions ...ata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
2
],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": {},
"HTTPDiffBodyProportionFactor": 1,
Expand All @@ -23,8 +25,6 @@
"HTTPFinalResponsesWithTLS": {
"4": true
},
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": {},
"TCPTransactionsWithUnexplainedUnexpectedFailures": {}
}
4 changes: 2 additions & 2 deletions ...onnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/analysis_classic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
2
],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": null,
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -14,8 +16,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": {}
}
4 changes: 2 additions & 2 deletions ...l/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": {},
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
4 changes: 2 additions & 2 deletions ...peline/testdata/webconnectivity/generated/badSSLWithWrongServerName/analysis_classic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"DNSLookupSuccessWithInvalidAddresses": [],
"DNSLookupSuccessWithInvalidAddressesClassic": [],
"DNSLookupUnexpectedFailure": [],
"TCPConnectUnexpectedFailure": [],
"TLSHandshakeUnexpectedFailure": [],
"DNSExperimentFailure": null,
"DNSPossiblyNonexistingDomains": null,
"HTTPDiffBodyProportionFactor": null,
Expand All @@ -10,8 +12,6 @@
"HTTPDiffUncommonHeadersIntersection": null,
"HTTPFinalResponsesWithControl": null,
"HTTPFinalResponsesWithTLS": null,
"TCPTransactionsWithUnexpectedTCPConnectFailures": {},
"TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
"TCPTransactionsWithUnexpectedHTTPFailures": null,
"TCPTransactionsWithUnexplainedUnexpectedFailures": null
}
Loading