-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: qatool to run webconnectivity lte QA (#1394)
This diff introduces the qatool, originally developed in #1392. This tool allows to run webconnectivitylte netemx-based QA tests and produces the measurement, observations, and analysis files. We can use this tool to run individual tests and inspect their results when they're failing. Additionally, we can use this tool to regenerate the test suite used by the minipipeline. When running the tool, we can select which QA tests to run, whether to rerun netemx and to reprocess. This feature is handy because we don't need to rerun netemx every time and sometimes we may only want to rerun netemx. This work is part of ooni/probe#2634. It helps us to manage and test the minipipeline and webconnectivitylte in an easier way, and it enables further development and testing.
- Loading branch information
1 parent
bb7dd9c
commit 9e84808
Showing
11 changed files
with
303 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "regexp" | ||
|
|
||
| "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" | ||
| "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivityqa" | ||
| "github.com/ooni/probe-cli/v3/internal/minipipeline" | ||
| "github.com/ooni/probe-cli/v3/internal/must" | ||
| "github.com/ooni/probe-cli/v3/internal/runtimex" | ||
| ) | ||
|
|
||
| var ( | ||
| // destdirFlag is the -destdir flag | ||
| destdirFlag = flag.String("destdir", "", "root directory in which to dump files") | ||
|
|
||
| // disableMeasureFlag is the -disable-measure flag | ||
| disableMeasureFlag = flag.Bool("disable-measure", false, "whether to measure again with netemx") | ||
|
|
||
| // disableReprocessFlag is the -disable-reprocess flag | ||
| disableReprocessFlag = flag.Bool("disable-reprocess", false, "whether to reprocess existing measurements") | ||
|
|
||
| // helpFlag is the -help flag | ||
| helpFlag = flag.Bool("help", false, "print help message") | ||
|
|
||
| // listFlag is the -list flag | ||
| listFlag = flag.Bool("list", false, "lists available tests") | ||
|
|
||
| // mustReadFileFn allows to overwrite must.ReadFile in tests | ||
| mustReadFileFn = must.ReadFile | ||
|
|
||
| // mustWriteFileFn allows to overwrite must.WriteFile in tests | ||
| mustWriteFileFn = must.WriteFile | ||
|
|
||
| // osExitFn allows to overwrite os.Exit in tests | ||
| osExitFn = os.Exit | ||
|
|
||
| // osMkdirAllFn allows to overwrite os.MkdirAll in tests | ||
| osMkdirAllFn = os.MkdirAll | ||
|
|
||
| // runFlag is the -run flag | ||
| runFlag = flag.String("run", "", "regexp to select which test cases to run") | ||
| ) | ||
|
|
||
| func mustSerializeMkdirAllAndWriteFile(dirname string, filename string, content any) { | ||
| rawData := must.MarshalAndIndentJSON(content, "", " ") | ||
| runtimex.Try0(osMkdirAllFn(dirname, 0700)) | ||
| mustWriteFileFn(filepath.Join(dirname, filename), rawData, 0600) | ||
| } | ||
|
|
||
| func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) { | ||
| // compute the actual destdir | ||
| actualDestdir := filepath.Join(*destdirFlag, tc.Name) | ||
|
|
||
| if !*disableMeasureFlag { | ||
| // construct the proper measurer | ||
| measurer := webconnectivitylte.NewExperimentMeasurer(&webconnectivitylte.Config{}) | ||
|
|
||
| // run the test case | ||
| measurement := runtimex.Try1(webconnectivityqa.MeasureTestCase(measurer, tc)) | ||
|
|
||
| // serialize the original measurement | ||
| mustSerializeMkdirAllAndWriteFile(actualDestdir, "measurement.json", measurement) | ||
| } | ||
|
|
||
| if !*disableReprocessFlag { | ||
| // obtain the web measurement | ||
| rawData := mustReadFileFn(filepath.Join(actualDestdir, "measurement.json")) | ||
| var webMeasurement minipipeline.WebMeasurement | ||
| must.UnmarshalJSON(rawData, &webMeasurement) | ||
|
|
||
| // ingest web measurement | ||
| observationsContainer := runtimex.Try1(minipipeline.IngestWebMeasurement(&webMeasurement)) | ||
|
|
||
| // serialize the observations | ||
| mustSerializeMkdirAllAndWriteFile(actualDestdir, "observations.json", observationsContainer) | ||
|
|
||
| // analyze the observations | ||
| analysis := minipipeline.AnalyzeWebObservations(observationsContainer) | ||
|
|
||
| // serialize the observations analysis | ||
| mustSerializeMkdirAllAndWriteFile(actualDestdir, "analysis.json", analysis) | ||
|
|
||
| // print the analysis to stdout | ||
| fmt.Printf("%s\n", must.MarshalAndIndentJSON(analysis, "", " ")) | ||
| } | ||
| } | ||
|
|
||
| func main() { | ||
| // parse command line flags | ||
| flag.Parse() | ||
|
|
||
| // print usage | ||
| if *helpFlag || (*destdirFlag == "" && !*listFlag) { | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "usage: %s -destdir <destdir> [-run <regexp>] [-disable-measure|-disable-reprocess]]\n", filepath.Base(os.Args[0])) | ||
| fmt.Fprintf(os.Stderr, " %s -list [-run <regexp>]\n", filepath.Base(os.Args[0])) | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "The first form of the command runs the QA tests selected by the given\n") | ||
| fmt.Fprintf(os.Stderr, "<regexp> and creates the corresponding files in <destdir>.\n") | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "The second form of the command lists the QA tests that would be run\n") | ||
| fmt.Fprintf(os.Stderr, "when using the given <regexp> selector.\n") | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "An empty <regepx> selector selects all QA tests.\n") | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "Add the -disable-measure flag to the first form of the command to\n") | ||
| fmt.Fprintf(os.Stderr, "avoid performing the measurements using netemx. This assums that\n") | ||
| fmt.Fprintf(os.Stderr, "you already generated the measurements previously.\n") | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| fmt.Fprintf(os.Stderr, "Add the -disable-reprocess flag to the first form of the command to\n") | ||
| fmt.Fprintf(os.Stderr, "avoid reprocessing the measurements using the minipipeline.\n") | ||
| fmt.Fprintf(os.Stderr, "\n") | ||
| osExitFn(1) | ||
| } | ||
|
|
||
| // build the regexp | ||
| selector := regexp.MustCompile(*runFlag) | ||
|
|
||
| // select which test cases to run | ||
| for _, tc := range webconnectivityqa.AllTestCases() { | ||
| name := "webconnectivitylte/" + tc.Name | ||
| if *runFlag != "" && !selector.MatchString(name) { | ||
| continue | ||
| } | ||
| if *listFlag { | ||
| fmt.Printf("%s\n", name) | ||
| continue | ||
| } | ||
| runWebConnectivityLTE(tc) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "io/fs" | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/google/go-cmp/cmp" | ||
| "github.com/ooni/probe-cli/v3/internal/runtimex" | ||
| ) | ||
|
|
||
| func TestMainList(t *testing.T) { | ||
| // reconfigure the global options for main | ||
| *destdirFlag = "" | ||
| *listFlag = true | ||
| mustReadFileFn = func(filename string) []byte { | ||
| panic(errors.New("mustReadFileFn")) | ||
| } | ||
| mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { | ||
| panic(errors.New("mustWriteFileFn")) | ||
| } | ||
| osExitFn = func(code int) { | ||
| panic(fmt.Errorf("osExit: %d", code)) | ||
| } | ||
| osMkdirAllFn = func(path string, perm os.FileMode) error { | ||
| panic(errors.New("osMkdirAllFn")) | ||
| } | ||
| *runFlag = "" | ||
|
|
||
| // run the main function | ||
| main() | ||
| } | ||
|
|
||
| func TestMainSuccess(t *testing.T) { | ||
| // reconfigure the global options for main | ||
| *destdirFlag = "xo" | ||
| *listFlag = false | ||
| contentmap := make(map[string][]byte) | ||
| mustReadFileFn = func(filename string) []byte { | ||
| data, found := contentmap[filename] | ||
| runtimex.Assert(found, fmt.Sprintf("cannot find %s", filename)) | ||
| return data | ||
| } | ||
| mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { | ||
| // make sure we can parse as JSON | ||
| var container map[string]any | ||
| if err := json.Unmarshal(content, &container); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| // register we have written a file | ||
| contentmap[filename] = content | ||
| } | ||
| osExitFn = os.Exit | ||
| osMkdirAllFn = func(path string, perm os.FileMode) error { | ||
| return nil | ||
| } | ||
| *runFlag = "dnsBlocking" | ||
|
|
||
| // run the main function | ||
| main() | ||
|
|
||
| // make sure we attempted to write the desired files | ||
| expect := map[string]bool{ | ||
| "xo/dnsBlockingAndroidDNSCacheNoData/measurement.json": true, | ||
| "xo/dnsBlockingAndroidDNSCacheNoData/observations.json": true, | ||
| "xo/dnsBlockingAndroidDNSCacheNoData/analysis.json": true, | ||
| "xo/dnsBlockingNXDOMAIN/measurement.json": true, | ||
| "xo/dnsBlockingNXDOMAIN/observations.json": true, | ||
| "xo/dnsBlockingNXDOMAIN/analysis.json": true, | ||
| } | ||
| got := make(map[string]bool) | ||
| for key := range contentmap { | ||
| got[key] = true | ||
| } | ||
| if diff := cmp.Diff(expect, got); diff != "" { | ||
| t.Fatal(diff) | ||
| } | ||
| } | ||
|
|
||
| func TestMainUsage(t *testing.T) { | ||
| // reconfigure the global options for main | ||
| *destdirFlag = "" | ||
| *listFlag = false | ||
| mustReadFileFn = func(filename string) []byte { | ||
| panic(errors.New("mustReadFileFn")) | ||
| } | ||
| mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { | ||
| panic(errors.New("mustWriteFileFn")) | ||
| } | ||
| osExitFn = func(code int) { | ||
| panic(fmt.Errorf("osExit: %d", code)) | ||
| } | ||
| osMkdirAllFn = func(path string, perm os.FileMode) error { | ||
| panic(errors.New("osMkdirAllFn")) | ||
| } | ||
| *runFlag = "" | ||
|
|
||
| // run the main function | ||
| var err error | ||
| func() { | ||
| // intercept panic caused by osExit or other panics | ||
| defer func() { | ||
| if r := recover(); r != nil { | ||
| err = r.(error) | ||
| } | ||
| }() | ||
|
|
||
| // run the main function with the given args | ||
| main() | ||
| }() | ||
|
|
||
| // make sure we've got the expected error | ||
| if err == nil || err.Error() != "osExit: 1" { | ||
| t.Fatal("expected", "os.Exit: 1", "got", err) | ||
| } | ||
| } |
Oops, something went wrong.