fix(minipipeline): set control expectation for all observations (#1419)

Up until now, we were setting the control expectation (i.e., the final result of accessing the website from the control) only for TCP endpoints observations. However, there are cases where we're considering DNS lookup failures where we also need this information. So, let's just make sure every observation we produce knows about what's the final expectation according to the test helper. Part of ooni/probe#2634
ooni · Nov 30, 2023 · 4dc6e7d · 4dc6e7d
1 parent 7447e43
commit 4dc6e7d
Show file tree

Hide file tree

Showing 65 changed files with 3,889 additions and 1,228 deletions.
diff --git a/internal/cmd/minipipeline/testdata/observations.json b/internal/cmd/minipipeline/testdata/observations.json
@@ -35,11 +35,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     },
     {
       "DNSTransactionID": 3,
@@ -76,11 +91,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
   ],
   "DNSLookupSuccesses": [
@@ -121,11 +151,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     },
     {
       "DNSTransactionID": 2,
@@ -164,11 +209,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     },
     {
       "DNSTransactionID": 3,
@@ -207,11 +267,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
   ],
   "KnownTCPEndpoints": {

diff --git a/internal/cmd/minipipeline/testdata/observations_classic.json b/internal/cmd/minipipeline/testdata/observations_classic.json
@@ -38,11 +38,26 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 36546,
+      "ControlHTTPResponseHeadersKeys": {
+        "Cache-Control": true,
+        "Content-Language": true,
+        "Content-Type": true,
+        "Date": true,
+        "Etag": true,
+        "Expires": true,
+        "Last-Modified": true,
+        "Link": true,
+        "Server": true,
+        "Vary": true,
+        "X-Content-Type-Options": true,
+        "X-Drupal-Cache": true,
+        "X-Frame-Options": true,
+        "X-Generator": true
+      },
+      "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
   ],
   "KnownTCPEndpoints": {

diff --git a/internal/minipipeline/observation.go b/internal/minipipeline/observation.go
@@ -537,7 +537,18 @@ func (c *WebObservationsContainer) controlXrefTLSFailures(resp *model.THResponse
 }
 
 func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *model.THResponse) {
+
+	// We need to set expectations for each type of observation. For example, to detect
+	// NXDOMAIN blocking with redirects when there's the expectation of success, we need
+	// to have the expectation inside the DNS-lookup-failure observation.
+	var observations []*WebObservation
+	observations = append(observations, c.DNSLookupFailures...)
+	observations = append(observations, c.DNSLookupSuccesses...)
 	for _, obs := range c.KnownTCPEndpoints {
+		observations = append(observations, obs)
+	}
+
+	for _, obs := range observations {
 		obs.ControlHTTPFailure = optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure))
 
 		// leave everything else nil if there was a failure, like we

diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -35,7 +35,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -80,7 +80,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -123,7 +123,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,

diff --git a/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json b/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json
@@ -38,7 +38,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,

diff --git a/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json b/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json
@@ -35,7 +35,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -76,7 +76,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -121,7 +121,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -164,7 +164,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,
@@ -207,7 +207,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,

diff --git a/...nectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json b/...nectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json
@@ -38,7 +38,7 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
+      "ControlHTTPFailure": "unknown_error",
       "ControlHTTPResponseStatusCode": null,
       "ControlHTTPResponseBodyLength": null,
       "ControlHTTPResponseHeadersKeys": null,

diff --git a/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json b/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json
@@ -35,11 +35,16 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 1533,
+      "ControlHTTPResponseHeadersKeys": {
+        "Alt-Svc": true,
+        "Content-Length": true,
+        "Content-Type": true,
+        "Date": true
+      },
+      "ControlHTTPResponseTitle": "Default Web Page"
     }
   ],
   "DNSLookupSuccesses": [
@@ -80,11 +85,16 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 1533,
+      "ControlHTTPResponseHeadersKeys": {
+        "Alt-Svc": true,
+        "Content-Length": true,
+        "Content-Type": true,
+        "Date": true
+      },
+      "ControlHTTPResponseTitle": "Default Web Page"
     },
     {
       "DNSTransactionID": 1,
@@ -123,11 +133,16 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 1533,
+      "ControlHTTPResponseHeadersKeys": {
+        "Alt-Svc": true,
+        "Content-Length": true,
+        "Content-Type": true,
+        "Date": true
+      },
+      "ControlHTTPResponseTitle": "Default Web Page"
     }
   ],
   "KnownTCPEndpoints": {

diff --git a/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json b/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json
@@ -38,11 +38,16 @@
       ],
       "ControlTCPConnectFailure": null,
       "ControlTLSHandshakeFailure": null,
-      "ControlHTTPFailure": null,
-      "ControlHTTPResponseStatusCode": null,
-      "ControlHTTPResponseBodyLength": null,
-      "ControlHTTPResponseHeadersKeys": null,
-      "ControlHTTPResponseTitle": null
+      "ControlHTTPFailure": "",
+      "ControlHTTPResponseStatusCode": 200,
+      "ControlHTTPResponseBodyLength": 1533,
+      "ControlHTTPResponseHeadersKeys": {
+        "Alt-Svc": true,
+        "Content-Length": true,
+        "Content-Type": true,
+        "Date": true
+      },
+      "ControlHTTPResponseTitle": "Default Web Page"
     }
   ],
   "KnownTCPEndpoints": {