From 9e84808d2f2af5160591dc4c9204e99041c96314 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 28 Nov 2023 12:02:57 +0100 Subject: [PATCH] feat: qatool to run webconnectivity lte QA (#1394) This diff introduces the qatool, originally developed in https://github.com/ooni/probe-cli/pull/1392. This tool allows to run webconnectivitylte netemx-based QA tests and produces the measurement, observations, and analysis files. We can use this tool to run individual tests and inspect their results when they're failing. Additionally, we can use this tool to regenerate the test suite used by the minipipeline. When running the tool, we can select which QA tests to run, whether to rerun netemx and to reprocess. This feature is handy because we don't need to rerun netemx every time and sometimes we may only want to rerun netemx. This work is part of https://github.com/ooni/probe/issues/2634. It helps us to manage and test the minipipeline and webconnectivitylte in an easier way, and it enables further development and testing. --- internal/cmd/minipipeline/main.go | 26 ++-- internal/cmd/minipipeline/main_test.go | 37 ++--- internal/cmd/qatool/main.go | 136 ++++++++++++++++++ internal/cmd/qatool/main_test.go | 120 ++++++++++++++++ internal/experiment/webconnectivityqa/run.go | 21 ++- .../experiment/webconnectivityqa/session.go | 2 +- .../dnsBlockingBOGON/observations.json | 2 +- .../analysis.json | 2 +- .../observations.json | 2 +- .../analysis.json | 4 +- .../observations.json | 2 +- 11 files changed, 303 insertions(+), 51 deletions(-) create mode 100644 internal/cmd/qatool/main.go create mode 100644 internal/cmd/qatool/main_test.go diff --git a/internal/cmd/minipipeline/main.go b/internal/cmd/minipipeline/main.go index 3c9edf496a..654c136a46 100644 --- a/internal/cmd/minipipeline/main.go +++ b/internal/cmd/minipipeline/main.go @@ -12,25 +12,25 @@ import ( ) var ( - // destdir is the -destdir flag - destdir = flag.String("destdir", ".", "destination directory to use") + // destdirFlag is the -destdir flag + destdirFlag = flag.String("destdir", ".", "destination directory to use") - // measurement is the -measurement flag - measurement = flag.String("measurement", "", "measurement file to analyze") + // measurementFlag is the -measurement flag + measurementFlag = flag.String("measurement", "", "measurement file to analyze") // mustWriteFileLn allows overwriting must.WriteFile in tests mustWriteFileFn = must.WriteFile - // prefix is the -prefix flag - prefix = flag.String("prefix", "", "prefix to add to generated files") + // prefixFlag is the -prefix flag + prefixFlag = flag.String("prefix", "", "prefix to add to generated files") - // osExit allows overwriting os.Exit in tests - osExit = os.Exit + // osExitFn allows overwriting os.Exit in tests + osExitFn = os.Exit ) func main() { flag.Parse() - if *measurement == "" { + if *measurementFlag == "" { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "usage: %s -measurement [-prefix ]\n", filepath.Base(os.Args[0])) fmt.Fprintf(os.Stderr, "\n") @@ -43,20 +43,20 @@ func main() { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Use -prefix to add in front of the generated files names.\n") fmt.Fprintf(os.Stderr, "\n") - osExit(1) + osExitFn(1) } // parse the measurement file var parsed minipipeline.WebMeasurement - must.UnmarshalJSON(must.ReadFile(*measurement), &parsed) + must.UnmarshalJSON(must.ReadFile(*measurementFlag), &parsed) // generate and write observations - observationsPath := filepath.Join(*destdir, *prefix+"observations.json") + observationsPath := filepath.Join(*destdirFlag, *prefixFlag+"observations.json") container := runtimex.Try1(minipipeline.IngestWebMeasurement(&parsed)) mustWriteFileFn(observationsPath, must.MarshalAndIndentJSON(container, "", " "), 0600) // generate and write observations analysis - analysisPath := filepath.Join(*destdir, *prefix+"analysis.json") + analysisPath := filepath.Join(*destdirFlag, *prefixFlag+"analysis.json") analysis := minipipeline.AnalyzeWebObservations(container) mustWriteFileFn(analysisPath, must.MarshalAndIndentJSON(analysis, "", " "), 0600) } diff --git a/internal/cmd/minipipeline/main_test.go b/internal/cmd/minipipeline/main_test.go index 9df67a88e5..0597cc6693 100644 --- a/internal/cmd/minipipeline/main_test.go +++ b/internal/cmd/minipipeline/main_test.go @@ -25,23 +25,15 @@ func mustloaddata(contentmap map[string][]byte, key string) (object map[string]a } func TestMainSuccess(t *testing.T) { - // make sure we set the destination directory - *destdir = "xo" - - // make sure we're reading from the expected input - *measurement = filepath.Join("testdata", "measurement.json") - - // make sure we store the expected output + // reconfigure the global options for main + *destdirFlag = "xo" + *measurementFlag = filepath.Join("testdata", "measurement.json") contentmap := make(map[string][]byte) mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { contentmap[filename] = content } - - // make sure osExit is correct - osExit = os.Exit - - // also check whether we can add a prefix - *prefix = "y-" + osExitFn = os.Exit + *prefixFlag = "y-" // run the main function main() @@ -62,25 +54,18 @@ func TestMainSuccess(t *testing.T) { } func TestMainUsage(t *testing.T) { - // make sure we clear the destination directory - *destdir = "" - - // make sure the expected input file is empty - *measurement = "" - - // make sure we panic if we try to write on disk + // reconfigure the global options for main + *destdirFlag = "" + *measurementFlag = "" mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { panic(errors.New("mustWriteFileFn")) } - - // make sure osExit is correct - osExit = func(code int) { + osExitFn = func(code int) { panic(fmt.Errorf("osExit: %d", code)) } + *prefixFlag = "" - // make sure the prefix is also clean - *prefix = "" - + // run the main function var err error func() { // intercept panic caused by osExit or other panics diff --git a/internal/cmd/qatool/main.go b/internal/cmd/qatool/main.go new file mode 100644 index 0000000000..0ea117168b --- /dev/null +++ b/internal/cmd/qatool/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivityqa" + "github.com/ooni/probe-cli/v3/internal/minipipeline" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ( + // destdirFlag is the -destdir flag + destdirFlag = flag.String("destdir", "", "root directory in which to dump files") + + // disableMeasureFlag is the -disable-measure flag + disableMeasureFlag = flag.Bool("disable-measure", false, "whether to measure again with netemx") + + // disableReprocessFlag is the -disable-reprocess flag + disableReprocessFlag = flag.Bool("disable-reprocess", false, "whether to reprocess existing measurements") + + // helpFlag is the -help flag + helpFlag = flag.Bool("help", false, "print help message") + + // listFlag is the -list flag + listFlag = flag.Bool("list", false, "lists available tests") + + // mustReadFileFn allows to overwrite must.ReadFile in tests + mustReadFileFn = must.ReadFile + + // mustWriteFileFn allows to overwrite must.WriteFile in tests + mustWriteFileFn = must.WriteFile + + // osExitFn allows to overwrite os.Exit in tests + osExitFn = os.Exit + + // osMkdirAllFn allows to overwrite os.MkdirAll in tests + osMkdirAllFn = os.MkdirAll + + // runFlag is the -run flag + runFlag = flag.String("run", "", "regexp to select which test cases to run") +) + +func mustSerializeMkdirAllAndWriteFile(dirname string, filename string, content any) { + rawData := must.MarshalAndIndentJSON(content, "", " ") + runtimex.Try0(osMkdirAllFn(dirname, 0700)) + mustWriteFileFn(filepath.Join(dirname, filename), rawData, 0600) +} + +func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) { + // compute the actual destdir + actualDestdir := filepath.Join(*destdirFlag, tc.Name) + + if !*disableMeasureFlag { + // construct the proper measurer + measurer := webconnectivitylte.NewExperimentMeasurer(&webconnectivitylte.Config{}) + + // run the test case + measurement := runtimex.Try1(webconnectivityqa.MeasureTestCase(measurer, tc)) + + // serialize the original measurement + mustSerializeMkdirAllAndWriteFile(actualDestdir, "measurement.json", measurement) + } + + if !*disableReprocessFlag { + // obtain the web measurement + rawData := mustReadFileFn(filepath.Join(actualDestdir, "measurement.json")) + var webMeasurement minipipeline.WebMeasurement + must.UnmarshalJSON(rawData, &webMeasurement) + + // ingest web measurement + observationsContainer := runtimex.Try1(minipipeline.IngestWebMeasurement(&webMeasurement)) + + // serialize the observations + mustSerializeMkdirAllAndWriteFile(actualDestdir, "observations.json", observationsContainer) + + // analyze the observations + analysis := minipipeline.AnalyzeWebObservations(observationsContainer) + + // serialize the observations analysis + mustSerializeMkdirAllAndWriteFile(actualDestdir, "analysis.json", analysis) + + // print the analysis to stdout + fmt.Printf("%s\n", must.MarshalAndIndentJSON(analysis, "", " ")) + } +} + +func main() { + // parse command line flags + flag.Parse() + + // print usage + if *helpFlag || (*destdirFlag == "" && !*listFlag) { + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "usage: %s -destdir [-run ] [-disable-measure|-disable-reprocess]]\n", filepath.Base(os.Args[0])) + fmt.Fprintf(os.Stderr, " %s -list [-run ]\n", filepath.Base(os.Args[0])) + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "The first form of the command runs the QA tests selected by the given\n") + fmt.Fprintf(os.Stderr, " and creates the corresponding files in .\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "The second form of the command lists the QA tests that would be run\n") + fmt.Fprintf(os.Stderr, "when using the given selector.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "An empty selector selects all QA tests.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "Add the -disable-measure flag to the first form of the command to\n") + fmt.Fprintf(os.Stderr, "avoid performing the measurements using netemx. This assums that\n") + fmt.Fprintf(os.Stderr, "you already generated the measurements previously.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "Add the -disable-reprocess flag to the first form of the command to\n") + fmt.Fprintf(os.Stderr, "avoid reprocessing the measurements using the minipipeline.\n") + fmt.Fprintf(os.Stderr, "\n") + osExitFn(1) + } + + // build the regexp + selector := regexp.MustCompile(*runFlag) + + // select which test cases to run + for _, tc := range webconnectivityqa.AllTestCases() { + name := "webconnectivitylte/" + tc.Name + if *runFlag != "" && !selector.MatchString(name) { + continue + } + if *listFlag { + fmt.Printf("%s\n", name) + continue + } + runWebConnectivityLTE(tc) + } +} diff --git a/internal/cmd/qatool/main_test.go b/internal/cmd/qatool/main_test.go new file mode 100644 index 0000000000..b1f231410c --- /dev/null +++ b/internal/cmd/qatool/main_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestMainList(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "" + *listFlag = true + mustReadFileFn = func(filename string) []byte { + panic(errors.New("mustReadFileFn")) + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + panic(errors.New("mustWriteFileFn")) + } + osExitFn = func(code int) { + panic(fmt.Errorf("osExit: %d", code)) + } + osMkdirAllFn = func(path string, perm os.FileMode) error { + panic(errors.New("osMkdirAllFn")) + } + *runFlag = "" + + // run the main function + main() +} + +func TestMainSuccess(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "xo" + *listFlag = false + contentmap := make(map[string][]byte) + mustReadFileFn = func(filename string) []byte { + data, found := contentmap[filename] + runtimex.Assert(found, fmt.Sprintf("cannot find %s", filename)) + return data + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + // make sure we can parse as JSON + var container map[string]any + if err := json.Unmarshal(content, &container); err != nil { + t.Fatal(err) + } + + // register we have written a file + contentmap[filename] = content + } + osExitFn = os.Exit + osMkdirAllFn = func(path string, perm os.FileMode) error { + return nil + } + *runFlag = "dnsBlocking" + + // run the main function + main() + + // make sure we attempted to write the desired files + expect := map[string]bool{ + "xo/dnsBlockingAndroidDNSCacheNoData/measurement.json": true, + "xo/dnsBlockingAndroidDNSCacheNoData/observations.json": true, + "xo/dnsBlockingAndroidDNSCacheNoData/analysis.json": true, + "xo/dnsBlockingNXDOMAIN/measurement.json": true, + "xo/dnsBlockingNXDOMAIN/observations.json": true, + "xo/dnsBlockingNXDOMAIN/analysis.json": true, + } + got := make(map[string]bool) + for key := range contentmap { + got[key] = true + } + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } +} + +func TestMainUsage(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "" + *listFlag = false + mustReadFileFn = func(filename string) []byte { + panic(errors.New("mustReadFileFn")) + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + panic(errors.New("mustWriteFileFn")) + } + osExitFn = func(code int) { + panic(fmt.Errorf("osExit: %d", code)) + } + osMkdirAllFn = func(path string, perm os.FileMode) error { + panic(errors.New("osMkdirAllFn")) + } + *runFlag = "" + + // run the main function + var err error + func() { + // intercept panic caused by osExit or other panics + defer func() { + if r := recover(); r != nil { + err = r.(error) + } + }() + + // run the main function with the given args + main() + }() + + // make sure we've got the expected error + if err == nil || err.Error() != "osExit: 1" { + t.Fatal("expected", "os.Exit: 1", "got", err) + } +} diff --git a/internal/experiment/webconnectivityqa/run.go b/internal/experiment/webconnectivityqa/run.go index e9c44f7777..b851e4d4cf 100644 --- a/internal/experiment/webconnectivityqa/run.go +++ b/internal/experiment/webconnectivityqa/run.go @@ -12,8 +12,8 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// RunTestCase runs a [testCase]. -func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { +// MeasureTestCase returns the JSON measurement produced by a [TestCase]. +func MeasureTestCase(measurer model.ExperimentMeasurer, tc *TestCase) (*model.Measurement, error) { // configure the netemx scenario env := netemx.MustNewScenario(netemx.InternetScenario) defer env.Close() @@ -34,7 +34,8 @@ func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { var err error env.Do(func() { // create an HTTP client inside the env.Do function so we're using netem - // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS + // but they're not needed here httpClient := netxlite.NewHTTPClientStdlib(prefixLogger) arguments := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(prefixLogger), @@ -54,9 +55,19 @@ func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { // handle the case of unexpected result switch { case err != nil && !tc.ExpectErr: - return fmt.Errorf("expected to see no error but got %s", err.Error()) + return nil, fmt.Errorf("expected to see no error but got %s", err.Error()) case err == nil && tc.ExpectErr: - return fmt.Errorf("expected to see an error but got ") + return nil, fmt.Errorf("expected to see an error but got ") + } + + return measurement, nil +} + +// RunTestCase runs a [testCase]. +func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { + measurement, err := MeasureTestCase(measurer, tc) + if err != nil { + return err } // reduce the test keys to a common format diff --git a/internal/experiment/webconnectivityqa/session.go b/internal/experiment/webconnectivityqa/session.go index 0abb31496a..5af5eb5147 100644 --- a/internal/experiment/webconnectivityqa/session.go +++ b/internal/experiment/webconnectivityqa/session.go @@ -6,7 +6,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/netemx" ) -// nwSession creates a new [model.ExperimentSession]. +// newSession creates a new [model.ExperimentSession]. func newSession(client model.HTTPClient, logger model.Logger) model.ExperimentSession { return &mocks.Session{ MockGetTestHelpersByName: func(name string) ([]model.OOAPIService, bool) { diff --git a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json index 7aee6a93ce..64bdeb18df 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json @@ -210,4 +210,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json index 067dafae93..9f5839859c 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json @@ -13,4 +13,4 @@ "TCPTransactionsWithUnexpectedTLSHandshakeFailures": {}, "TCPTransactionsWithUnexpectedHTTPFailures": {}, "TCPTransactionsWithUnexplainedUnexpectedFailures": {} -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json index 3b200cb5ea..993dc6594a 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json @@ -242,4 +242,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json index b47650dfa1..7fdae21358 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json @@ -3,7 +3,7 @@ "DNSTransactionsWithBogons": {}, "DNSTransactionsWithUnexpectedFailures": {}, "DNSPossiblyInvalidAddrs": { - "83.224.65.41": true + "83.224.65.41": true }, "HTTPDiffBodyProportionFactor": 1, "HTTPDiffStatusCodeMatch": true, @@ -22,4 +22,4 @@ "TCPTransactionsWithUnexpectedTLSHandshakeFailures": {}, "TCPTransactionsWithUnexpectedHTTPFailures": {}, "TCPTransactionsWithUnexplainedUnexpectedFailures": {} -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json index 156923e8d8..3f72ac998f 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json @@ -296,4 +296,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file