Hook up playos-basic e2e test with pyppeteer and test web storage per…

…sistence

...which currently does not seem to work properly?

testing/end-to-end/default.nix

@@ -1,4 +1,4 @@
-{pkgs, disk, safeProductName, updateUrl, ...}:
+{pkgs, disk, safeProductName, updateUrl, kioskUrl, ...}:
 with builtins;
 with pkgs.lib;
 let
@@ -8,7 +8,7 @@ let
     testPackages = map
         (file: pkgs.callPackage file
             # TODO: why does (args // {inherit overlayPath}) not work??
-            { inherit overlayPath disk safeProductName updateUrl; }
+            { inherit overlayPath disk safeProductName updateUrl kioskUrl; }
         )
         (fileset.toList testFiles);
     # TODO: Currently this builds AND runs the tests, however

testing/end-to-end/test-script-helpers.py

@@ -15,6 +15,8 @@ def create_overlay(disk, overlay_path):
         ],
         check=True)
 
+# TODO: upgrade this into a magic proxy class that makes every
+# assert* call loggable, to make debugging easier
 class TestCase(object):
     def __init__(self, test_descr):
         self.test_descr = test_descr
@@ -42,6 +44,7 @@ def __exit__(self, exc_type, exc_value, traceback):
 
         return False # signals to re-raise the exception
 
+
 def wait_for_logs(vm, regex, unit=None, timeout=10):
     maybe_unit = f"--unit={unit}" if unit else ""
     journal_cmd = f"journalctl --reverse {maybe_unit}"
@@ -52,3 +55,36 @@ def wait_for_logs(vm, regex, unit=None, timeout=10):
         print("Last VM logs:\n")
         print(output)
         raise e
+
+
+def expose_local_port(vm, port):
+    # forward external `port` to 127.0.0.1:port to allow external
+    # access to internal services in PlayOS
+
+    # enable NAT on loopback
+    vm.succeed("sysctl net.ipv4.conf.all.route_localnet=1")
+
+    # forward the port
+    vm.succeed("iptables -t nat -A PREROUTING -p tcp " + \
+                f"--dport {port} -j DNAT --to-destination 127.0.0.1:{port}")
+
+    # open the port in the firewall
+    vm.succeed(f"iptables -A INPUT -p tcp --dport {port} -j ACCEPT")
+    vm.succeed("systemctl reload firewall")
+
+# with exponential back-off
+async def retry_until_no_exception(func, retries=3, sleep=3.0):
+    import asyncio
+    total_retries = retries
+    while True:
+        retries -= 1
+        try:
+            return (await func())
+        except Exception as e:
+            if (retries > 0):
+                print(f"Func failed with {e}, sleeping for {sleep} seconds")
+                await asyncio.sleep(sleep)
+                sleep *= 2
+            else:
+                print(f"Func failed with {e}, giving up after {total_retries}")
+                raise e

testing/end-to-end/tests/playos-basic.nix

@@ -1,4 +1,9 @@
-{pkgs, disk, overlayPath, ...}:
+{pkgs, disk, overlayPath, kioskUrl, ...}:
+let
+    # TODO: set/get in application.nix
+    guestCDPport = 3355;
+    hostCDPport = 13355;
+in
 pkgs.testers.runNixOSTest {
   name = "Built PlayOS is functional";
 
@@ -8,55 +13,168 @@ pkgs.testers.runNixOSTest {
       imports = [
         (import ../virtualisation-config.nix { inherit overlayPath; })
       ];
+      config = {
+        virtualisation.forwardPorts = [
+            {   from = "host";
+                # TODO: would be nicer to get a random port instead
+                host.port = hostCDPport;
+                guest.port = guestCDPport;
+            }
+        ];
+      };
+
     };
   };
 
-  extraPythonPackages = ps: [ps.types-colorama];
+  extraPythonPackages = ps: [
+    ps.types-colorama
+    ps.pyppeteer
+    ps.requests
+    ps.types-requests
+  ];
 
   testScript = ''
-    ${builtins.readFile ../test-script-helpers.py}
-    import json
+${builtins.readFile ../test-script-helpers.py}
+import json
+import requests
+import pyppeteer # type: ignore
+import asyncio
 
-    create_overlay("${disk}", "${overlayPath}")
+aio = asyncio.Runner()
 
-    playos.start(allow_reboot=True)
+create_overlay("${disk}", "${overlayPath}")
 
-    with TestCase("PlayOS disk boots"):
-        playos.wait_for_unit('multi-user.target')
-        playos.wait_for_x()
+playos.start(allow_reboot=True)
 
-    with TestCase("PlayOS services are runnning"):
-        playos.wait_for_unit('dividat-driver.service')
-        playos.wait_for_unit('playos-controller.service')
-        playos.wait_for_unit('playos-status.service')
+with TestCase("PlayOS disk boots"):
+    playos.wait_for_unit('multi-user.target')
+    playos.wait_for_x()
 
-    # TODO: add test to check that we can log into play.dividat.com
-    # TODO: add test to check that controller GUI works
+with TestCase("PlayOS services are runnning"):
+    playos.wait_for_unit('dividat-driver.service')
+    playos.wait_for_unit('playos-controller.service')
+    playos.wait_for_unit('playos-status.service')
 
-    with TestCase("Booted from system.a") as t:
-        rauc_status = json.loads(playos.succeed("rauc status --output-format=json"))
-        t.assertEqual(
-            rauc_status['booted'],
-            "a"
-        )
+async def connect_to_kiosk_debug_engine():
+    # TODO: enable runtime configuration of the remote debugging URL instead
+    expose_local_port(playos, ${toString guestCDPport})
 
-    # mark other (b) slot as active and try to reboot into it
-    playos.succeed('busctl call de.pengutronix.rauc / de.pengutronix.rauc.Installer Mark ss "active" "other"')
+    async def try_connect():
+        # connect to Qt WebEngine's CDP debug port
+        host_cdp_url = "http://127.0.0.1:${toString hostCDPport}/json/version"
+        cdp_info = requests.get(host_cdp_url).json()
+        ws_url = cdp_info['webSocketDebuggerUrl']
+        browser = await pyppeteer.launcher.connect(browserWSEndpoint=ws_url)
+        return browser
 
-    # NOTE: 'systemctl reboot' fails because of some bug in test-driver
-    # - it seems to keep consuming console text after the reboot
-    playos.shutdown()
-    playos.start()
+    # Sometimes I get connection reset by peer when connecting, I am guessing
+    # due to firewall restart when doing `expose_local_port`? Hence the retry.
+    return await retry_until_no_exception(try_connect)
 
-    # it should now boot without requiring to select anything in GRUB
-    playos.wait_for_x()
 
-    with TestCase("Booted into other slot") as t:
-        rauc_status = json.loads(playos.succeed("rauc status --output-format=json"))
-        t.assertEqual(
-            rauc_status['booted'],
-            "b",
-            "Did not boot from other (i.e. system.b) slot"
-        )
-  '';
+# Helper used to check web storage persistance after a reload or
+# a reboot ("hard" reload)
+async def check_web_storages_after_reload(page, t, hard_reload=False):
+    ss = await page.evaluate('sessionStorage.getItem("TEST_KEY")')
+    ls = await page.evaluate('localStorage.getItem("TEST_KEY")')
+    # Note: CDP automatically awaits the JS call
+    cs = await page.evaluate('cookieStore.get("TEST_KEY")')
+
+    # TODO: do we care about sessionStorage, actually?
+    expected_ss_val = None if hard_reload else "TEST_VALUE"
+    t.assertEqual(
+        ss,
+        expected_ss_val,
+        "Session store did not contain expected TEST_KEY value:" + \
+            f"(found {ss}, expected: {expected_ss_val})"
+    )
+
+    # localStorage and cookieStore should be persisted
+    t.assertEqual(
+        ls, "TEST_VALUE",
+        "TEST_KEY was not persisted in localStorage"
+    )
+    t.assertIn("value", cs,
+        "Cookie store did not return a value"
+    )
+    t.assertEqual(
+        cs['value'], "TEST_VALUE",
+        "TEST_KEY was not persisted in cookieStore"
+    )
+
+
+async def wait_for_kiosk_page(browser):
+    pages = await browser.pages()
+    t.assertEqual(
+        len(pages), 1,
+        f"Expected 1 browser page, found: {[page.url for page in pages]}"
+    )
+
+    page = pages[0]
+
+    t.assertIn(
+        "${kioskUrl}".rstrip("/"),
+        page.url,
+        "kiosk is not open with ${kioskUrl}"
+    )
+    return page
+
+with TestCase("Kiosk is open, web storage works") as t:
+    browser = aio.run(connect_to_kiosk_debug_engine())
+
+    # The retry is here because it seems that Qt WebEngine is reloading
+    # something after the firewall reload and it takes a while until
+    # it actually loads the kiosk page.
+    page = aio.run(retry_until_no_exception(
+        lambda: wait_for_kiosk_page(browser),
+        retries=5
+    ))
+
+    # store something in all the web storage engines
+    async def populate_web_storages():
+        await page.evaluate('sessionStorage.setItem("TEST_KEY", "TEST_VALUE")')
+        await page.evaluate('localStorage.setItem("TEST_KEY", "TEST_VALUE")')
+        await page.evaluate('cookieStore.set("TEST_KEY", "TEST_VALUE")')
+
+    aio.run(populate_web_storages())
+
+    aio.run(page.reload())
+
+    # mostly a sanity check, real check after reboot
+    aio.run(check_web_storages_after_reload(page, t))
+
+with TestCase("Booted from system.a") as t:
+    rauc_status = json.loads(playos.succeed("rauc status --output-format=json"))
+    t.assertEqual(
+        rauc_status['booted'],
+        "a"
+    )
+
+# mark other (b) slot as active and try to reboot into it
+playos.succeed(
+    'busctl call de.pengutronix.rauc / ' + \
+        'de.pengutronix.rauc.Installer Mark ss "active" "other"'
+)
+
+# NOTE: 'systemctl reboot' fails because of some bug in test-driver
+playos.shutdown()
+playos.start()
+
+playos.wait_for_x()
+
+with TestCase("Booted into other slot") as t:
+    rauc_status = json.loads(playos.succeed("rauc status --output-format=json"))
+    t.assertEqual(
+        rauc_status['booted'],
+        "b",
+        "Did not boot from other (i.e. system.b) slot"
+    )
+
+# TODO: currently fails ??!
+#with TestCase("kiosk's web storage is restored") as t:
+#    browser = aio.run(connect_to_kiosk_debug_engine())
+#    page = aio.run(retry_until_no_exception(lambda: wait_for_kiosk_page(browser)))
+#    aio.run(check_web_storages_after_reload(page, t, hard_reload=True))
+'';
+
 }