Add dynamic device passthrough functionality

- Add new host service for dynamic device passthrough
- Add vhotplug app
- Add qemu-xhci to vm's config
- Modify fmo-tool for dynamic device passthrough
- Minor modification

Signed-off-by: Anh Huy Bui <buianhhuy96@gmail.com>

hardware/fmo-os-rugged-laptop-7330.nix

@@ -49,6 +49,9 @@
               netvm = "/var/netvm/netconf/dpf.config";
             };
           }; # services.dynamic-portforwarding-service
+          fmo-dynamic-device-passthrough-service-host = {
+            enable = true;
+          }; # services.dynamic-device-passthrough-service-host
           fmo-config = {
             enable = true;
           }; # fmo-config
@@ -264,8 +267,8 @@
               devices = [
                 {
                   bus = "usb";
-                  vendorid = "0x1546";
-                  productid = "0x01a9";
+                  vendorid = "1546";
+                  productid = "01a9";
                 }
               ];
             }; # services.fmo-dynamic-device-passthrough

hardware/fmo-os-rugged-tablet-7230.nix

@@ -49,6 +49,9 @@
               netvm = "/var/netvm/netconf/dpf.config";
             };
           }; # services.dynamic-portforwarding-service
+          fmo-dynamic-device-passthrough-service-host = {
+            enable = true;
+          }; # services.dynamic-device-passthrough-service-host
           fmo-config = {
             enable = true;
           }; # fmo-config
@@ -239,11 +242,12 @@
               hostname-path = "/var/lib/fogdata/hostname";
             }; # services.fmo-hostnam-service
             fmo-dynamic-device-passthrough = {
+              enable = true;
               devices = [
                 {
                   bus = "usb";
-                  vendorid = "0x1546";
-                  productid = "0x01a9";
+                  vendorid = "1546";
+                  productid = "01a9";
                 }
               ];
             }; # services.fmo-dynamic-device-passthrough

modules/fmo-services/default.nix

@@ -14,5 +14,6 @@
     ./psk-distribution-vm
     ./registration-agent-laptop
     ./dynamic-device-passthrough-services
+    ./dynamic-device-passthrough-services-host
   ];
 }

modules/fmo-services/dynamic-device-passthrough-services-host/default.nix

@@ -0,0 +1,34 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.services.fmo-dynamic-device-passthrough-service-host;
+in {
+  options.services.fmo-dynamic-device-passthrough-service-host = {
+    enable = mkEnableOption "FMO dynamic device passthrough service";
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.vhotplug ];
+
+    services.udev.extraRules = ''
+      SUBSYSTEM=="usb", GROUP="kvm"
+      KERNEL=="event*", GROUP="kvm"
+    '';
+
+    systemd.services."fmo-dynamic-device-passthrough-service" = {
+      script = ''
+        if ! [ -f /var/host/vmddp.conf ]; then
+          ${pkgs.fmo-tool}/bin/fmo-tool ddp generate
+        fi
+        ${pkgs.vhotplug}/bin/vhotplug -a -c /var/host/vmddp.conf
+      '';
+      serviceConfig = {
+        Type = "simple";
+        RemainAfterExit = true;
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+  };
+}

modules/fmo-services/dynamic-device-passthrough-services/default.nix

@@ -6,7 +6,7 @@ let
   cfg = config.services.fmo-dynamic-device-passthrough;
 in {
   options.services.fmo-dynamic-device-passthrough = {
-    enable = mkEnableOption "FMO dynamic device passthrough service";
+    enable = mkEnableOption "FMO dynamic device passthrough devices";
 
     devices = mkOption {
       type = types.listOf types.attrs;
@@ -20,8 +20,4 @@ in {
       '';
     };
   };
-
-  config = mkIf cfg.enable {
-
-  };
 }

modules/installers/pterm-installer/default.nix

@@ -78,7 +78,7 @@ in
           rev = "0a12c7f3288f7019adc7781310f02f47f61444f1";
           ref = "refs/heads/main";
         };
-        vendorHash = "sha256-MKMsvIP8wMV86dh9Y5CWhgTQD0iRpzxk7+0diHkYBUo=";
+        vendorHash = "sha256-CFFSaURE4lJZS6E8XT6KcRv4hk6wO059lzD1V2wfZgI=";
         proxyVendor=true;
         ldflags = [
           "-X 'ghaf-installer/global.OSSfile=${cfg.oss_path}'"

modules/packages/default.nix

@@ -14,5 +14,6 @@ _: {
     (import ./sway-scripts)
     (import ./terminator)
     (import ./fmo-tool)
+    (import ./vhotplug)
   ];
 }

modules/packages/fmo-tool/ddp.patch

@@ -0,0 +1,150 @@
+diff --git a/apps/devicepassthrough_apps.py b/apps/devicepassthrough_apps.py
+index 5a4d43c..55ed422 100644
+--- a/apps/devicepassthrough_apps.py
++++ b/apps/devicepassthrough_apps.py
+@@ -1,15 +1,18 @@
+ import typer
++import os
+
+ from enum import Enum
+ from utils.ssh import ssh_vm_helper
+ from utils.misc import print_config, eprint
+ from utils.utils import get_ctx_if_vm_exist
++from utils.fmoconfig import get_fmoconfig_manager
+
+ from typing_extensions import Annotated
+
+
+ app = typer.Typer()
+
++SEVICE_NAME = "fmo-dynamic-device-passthrough-service"
+
+ class DP_Proto(str, Enum):
+     usb = "usb"
+@@ -32,6 +35,14 @@ def get_dp_config(
+
+     print(print_config(pfconfig))
+
++@app.command("generate")
++def generate_dp_config(
++) -> None:
++    """
++    Generate VM DDP config
++    """
++    ctx = get_fmoconfig_manager()
++    ctx.write_vmddp_config()
+
+ @app.command("add")
+ def add_dp_rules(
+@@ -47,6 +58,7 @@ def add_dp_rules(
+     ctx = get_ctx_if_vm_exist(vmname)
+     dpconfig = ctx.get_vmddp_config(vmname)
+     configuration = dpconfig.get("devices", [])
++    enabled = dpconfig.get("enable", None)
+
+     if bus == 'pcie':
+         eprint("PCIe dynamic device passthroug not implemented yet")
+@@ -62,8 +74,14 @@ def add_dp_rules(
+                'productid': productid, 'vendorid': vendorid}
+     configuration.append(newrule)
+     dpconfig['devices'] = configuration
++    if enabled == None:
++        dpconfig['enable'] = True
++    if enabled == False:
++        eprint('Set enable status using\n'\
++            'fmo-tool ddp enabled '+vmname+ ' --enable')
+     ctx.set_vmddp_config(vmname, dpconfig)
+     ctx.save_config()
++    os.system(f"sudo systemctl restart {SEVICE_NAME}.service")
+     raise typer.Exit(code=0)
+
+
+@@ -88,6 +106,7 @@ def delete_dp_rules(
+     eprint(f"Delete rule: {_}")
+     ctx.set_vmddp_config(vmname, dpconfig)
+     ctx.save_config()
++    os.system(f"sudo systemctl restart {SEVICE_NAME}.service")
+     raise typer.Exit(code=0)
+
+
+@@ -141,4 +160,5 @@ def ddp_service_enabled_cmd(
+         ctx.set_vmddp_enabled(vmname, enable)
+         ctx.save_config()
+
++    os.system(f"sudo systemctl restart {SEVICE_NAME}.service")
+     print(ctx.get_vmddp_enabled(vmname))
+diff --git a/utils/fmoconfig.py b/utils/fmoconfig.py
+index f961e09..cbc3a46 100644
+--- a/utils/fmoconfig.py
++++ b/utils/fmoconfig.py
+@@ -1,5 +1,6 @@
+ import os
+ import yaml
++import json
+
+ from typing import Dict, List
+ from utils.misc import eprint
+@@ -21,16 +22,18 @@ class FMOConfig_Manager(object, metaclass=Singleton):
+     __valid = False
+     __config_path_rw = ""
+     __config_path_ro = ""
++    __ddp_path_wo = ""
+
+     __DPF = "dynamic-portforwarding-service"
+     __DDP = "fmo-dynamic-device-passthrough"
+     __MON = "monitoring-service"
+
+-    def __init__(self, config_path_ro: str, config_path_rw: str):
++    def __init__(self, config_path_ro: str, config_path_rw: str, __ddp_path_wo: str):
+         self.__valid = False
+         self.__schema = None
+         self.__config_path_rw = config_path_rw
+         self.__config_path_ro = config_path_ro
++        self.__ddp_path_wo = __ddp_path_wo
+
+         try:
+             with open(config_path_rw) as yaml_conifg:
+@@ -69,6 +72,7 @@ class FMOConfig_Manager(object, metaclass=Singleton):
+         os.makedirs(os.path.dirname(self.__config_path_rw), exist_ok=True)
+         with open(self.__config_path_rw, "w") as yaml_config:
+             yaml.dump(config, yaml_config)
++        self.write_vmddp_config()
+
+     def restore_config(self) -> None:
+         import shutil
+@@ -155,6 +159,25 @@ class FMOConfig_Manager(object, metaclass=Singleton):
+
+         for k in newconf.keys():
+             config[k] = newconf[k]
++    
++    def write_vmddp_config(self) -> None:
++        vmlist = self.get_vms_names_list()
++        vmddplist = []
++        for vmname in vmlist:
++            dpconfig = self._get_vmddp_config(vmname).get_config()
++            try:
++                if (dpconfig.get("enable",False)):                    
++                    devicelist = dpconfig.get("devices", [])
++                    for device in devicelist:
++                        device["vendorId"]=device.pop("vendorid")
++                        device["productId"]=device.pop("productid")
++                    vmddplist.append({'name': vmname, 'qmpSocket': '/var/lib/microvms/'+vmname+'/'+vmname+'.sock', 'usbPassthrough': devicelist})
++            except Exception as e:
++                continue 
++        vhotplugconf = {"vms": vmddplist }
++        os.makedirs(os.path.dirname(self.__ddp_path_wo), exist_ok=True)
++        with open(self.__ddp_path_wo, "w") as ddp_config:  
++            json.dump(vhotplugconf, ddp_config)
+
+ # ############################
+ # # Internal usage functions #
+@@ -187,7 +210,8 @@ class FMOConfig_Manager(object, metaclass=Singleton):
+
+ def get_fmoconfig_manager() -> FMOConfig_Manager:
+     manager = FMOConfig_Manager("/etc/fmo-config.yaml",
+-                                "/var/host/fmo-config.yaml")
++                                "/var/host/fmo-config.yaml",
++                                "/var/host/vmddp.conf")
+     if not manager.is_valid():
+         eprint("Config is not valid")
+         raise Exception("Config is not valid")

modules/packages/fmo-tool/fmo-tool.nix

@@ -34,6 +34,7 @@ pkgs.python310Packages.buildPythonApplication {
     rich
   ];
 
+  patches = [ ./ddp.patch ];
   src = builtins.fetchGit {
     url = "git@github.com:tiiuae/fmo-tool.git";
     rev = "e797aad27578316945daa0b133058c1e753119ac";

modules/packages/vhotplug/default.nix

@@ -0,0 +1,5 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+(final: _prev: {
+  vhotplug = final.callPackage ./vhotplug.nix {};
+})

modules/packages/vhotplug/qemuqmp.nix

@@ -0,0 +1,28 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{
+  python3Packages,
+  fetchPypi,
+  lib,
+}:
+python3Packages.buildPythonPackage rec {
+  pname = "qemu.qmp";
+  version = "0.0.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-y8iPvMEV7pQ9hER9FyxkLaEgIgRRQWwvYhrPM98eEBA=";
+  };
+
+  pyproject = true;
+
+  nativeBuildInputs = [
+    python3Packages.setuptools-scm
+  ];
+
+  meta = {
+    homepage = "https://www.qemu.org/";
+    description = "QEMU Monitor Protocol library";
+    license = lib.licenses.lgpl2Plus;
+  };
+}

modules/packages/vhotplug/vhotplug.nix

@@ -0,0 +1,28 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{
+  python3Packages,
+  pkgs,
+  fetchFromGitHub,
+}: let
+  qemuqmp = pkgs.callPackage ./qemuqmp.nix {};
+in
+  python3Packages.buildPythonApplication rec {
+    pname = "vhotplug";
+    version = "0.1";
+
+    propagatedBuildInputs = [
+      python3Packages.pyudev
+      python3Packages.psutil
+      qemuqmp
+    ];
+
+    doCheck = false;
+
+    src = fetchFromGitHub {
+      owner = "tiiuae";
+      repo = "vhotplug";
+      rev = "fd05361ed893d06cdb5ac4a538c171e4a86b6f5a";
+      hash = "sha256-6fl5xeSpcIIBKn3dZUAEHiNRRpn9LbYC4Imap5KBH2M=";
+    };
+  }

modules/virtualization/microvm/vm.nix

@@ -33,6 +33,7 @@
         nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system;
 
         microvm.hypervisor = "qemu";
+        microvm.optimize.enable = false;
 
         networking = {
           enableIPv6 = false;
@@ -61,6 +62,12 @@
             mountPoint = "/nix/.ro-store";
          }
         ]; # microvm.shares
+
+        microvm.qemu.extraArgs = [
+          "-device"
+          "qemu-xhci"
+        ];
+
         microvm.writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";
 
         networking.nat = {