New experiment functionality, improved UI and various other improveme…

…nts and fixes (#38)

- New experiment functionality:
  - Dynamic python back-end scripting
  - User interaction simulation
  - **All** requests received during an experiment are stored
- New UI functionality:
  - Removing data points by SHIFT clicking
  - New experiment creation options
- Various improvements and bug fixes

.devcontainer/devcontainer.json

@@ -24,7 +24,10 @@
 				"Vue.volar"
 			]
 		}
-	}
+	},
+
+	// Install pip requirements
+	"postCreateCommand": "pip install -r requirements.txt"
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	// "features": {},
@@ -35,9 +38,6 @@
 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
 	// "shutdownAction": "none",
 
-	// Uncomment the next line to run commands after the container is created.
-	// "postCreateCommand": "cat /etc/os-release",
-
 	// Configure tool-specific properties.
 	// "customizations": {},
 }

.gitignore

@@ -6,6 +6,17 @@ nginx/ssl/keys/*
 !**/.gitkeep
 **/node_modules
 **/junit.xml
+
+# Screenshots
+logs/screenshots/*
+!logs/screenshots/.gitkeep
+
+# Fish shell
+$HOME
+
+# JetBrains IDEs
+.idea
+
 # Created by https://www.toptal.com/developers/gitignore/api/intellij,python,flask,macos
 # Edit at https://www.toptal.com/developers/gitignore?templates=intellij,python,flask,macos
 

Dockerfile

@@ -54,7 +54,11 @@ RUN cp /app/scripts/daemon/xvfb /etc/init.d/xvfb
 # Install python packages
 COPY requirements.txt /app/requirements.txt
 RUN pip install --user -r /app/requirements.txt
+RUN apt-get install python3-tk python3-xlib gnome-screenshot -y
 
+# Initiate PyAutoGUI
+RUN touch /root/.Xauthority && \
+    xauth add ${HOST}:0 . $(xxd -l 16 -p /dev/urandom)
 
 FROM base AS core
 # Copy rest of source code

bci/analysis/plot_factory.py

@@ -5,14 +5,14 @@
 class PlotFactory:
 
     @staticmethod
-    def get_plot_revision_data(params: PlotParameters, db: MongoDB) -> dict:
-        revision_docs = db.get_documents_for_plotting(params)
+    def get_plot_revision_data(params: PlotParameters) -> dict:
+        revision_docs = MongoDB().get_documents_for_plotting(params)
         revision_results = PlotFactory.__add_outcome_info(params, revision_docs)
         return revision_results
 
     @staticmethod
-    def get_plot_version_data(params: PlotParameters, db: MongoDB) -> dict:
-        version_docs = db.get_documents_for_plotting(params, releases=True)
+    def get_plot_version_data(params: PlotParameters) -> dict:
+        version_docs = MongoDB().get_documents_for_plotting(params, releases=True)
         version_results = PlotFactory.__add_outcome_info(params, version_docs)
         return version_results
 

bci/app.py

@@ -3,28 +3,37 @@
 from flask import Flask
 from flask_sock import Sock
 
-from bci.main import Main as bci_api
+from bci.configuration import Global, Loggers
+from bci.main import Main
 
 sock = Sock()
 
+
 def create_app():
-    bci_api.initialize()
+    Loggers.configure_loggers()
+
+    if not Global.check_required_env_parameters():
+        raise Exception('Not all required environment variables are available')
+
+    # Instantiate main object and add to global flask context
+    main = Main()
 
     # Blueprint modules are only imported after loggers are configured
     from bci.web.blueprints.api import api
     from bci.web.blueprints.experiments import exp
 
     app = Flask(__name__)
     # We don't store anything sensitive in the session, so we can use a simple secret key
+    app.config['main'] = main
     app.secret_key = 'secret_key'
 
     app.register_blueprint(api)
     app.register_blueprint(exp)
     sock.init_app(app)
 
     # Configure signal handlers
-    signal.signal(signal.SIGTERM, bci_api.sigint_handler)
-    signal.signal(signal.SIGINT, bci_api.sigint_handler)
+    signal.signal(signal.SIGTERM, main.sigint_handler)
+    signal.signal(signal.SIGINT, main.sigint_handler)
 
     return app
 

bci/browser/automation/terminal.py

@@ -7,31 +7,35 @@
 
 
 class TerminalAutomation:
-
     @staticmethod
-    def run(url: str, args: list[str], seconds_per_visit: int):
-        logger.debug("Starting browser process...")
+    def visit_url(url: str, args: list[str], seconds_per_visit: int):
         args.append(url)
+        proc = TerminalAutomation.open_browser(args)
+        logger.debug(f'Visiting the page for {seconds_per_visit}s')
+        time.sleep(seconds_per_visit)
+        TerminalAutomation.terminate_browser(proc, args)
+
+    @staticmethod
+    def open_browser(args: list[str]) -> subprocess.Popen:
+        logger.debug('Starting browser process...')
         logger.debug(f'Command string: \'{" ".join(args)}\'')
-        with open('/tmp/browser.log', 'a') as file:
-            proc = subprocess.Popen(
-                args,
-                stdout=file,
-                stderr=file
-            )
+        with open('/tmp/browser.log', 'a+') as file:
+            proc = subprocess.Popen(args, stdout=file, stderr=file)
+            return proc
 
-        time.sleep(seconds_per_visit)
+    @staticmethod
+    def terminate_browser(proc: subprocess.Popen, args: list[str]) -> None:
+        logger.debug('Terminating browser process using SIGINT...')
 
-        logger.debug(f'Terminating browser process after {seconds_per_visit}s using SIGINT...')
         # Use SIGINT and SIGTERM to end process such that cookies remain saved.
         proc.send_signal(signal.SIGINT)
         proc.send_signal(signal.SIGTERM)
 
         try:
             stdout, stderr = proc.communicate(timeout=5)
         except subprocess.TimeoutExpired:
-            logger.info("Browser process did not terminate after 5s. Killing process through pkill...")
+            logger.info('Browser process did not terminate after 5s. Killing process through pkill...')
             subprocess.run(['pkill', '-2', args[0].split('/')[-1]])
 
         proc.wait()
-        logger.debug("Browser process terminated.")
+        logger.debug('Browser process terminated.')

bci/browser/configuration/browser.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import subprocess
 from abc import abstractmethod
 
 import bci.browser.binary.factory as binary_factory
@@ -15,9 +16,13 @@
 
 
 class Browser:
+    process: subprocess.Popen | None
 
-    def __init__(self, browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, binary: Binary) -> None:
+    def __init__(
+        self, browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, binary: Binary
+    ) -> None:
         self.browser_config = browser_config
+        self.process = None
         self.eval_config = eval_config
         self.binary = binary
         self.state = binary.state
@@ -34,10 +39,22 @@ def visit(self, url: str):
         match self.eval_config.automation:
             case 'terminal':
                 args = self._get_terminal_args()
-                TerminalAutomation.run(url, args, self.eval_config.seconds_per_visit)
+                TerminalAutomation.visit_url(url, args, self.eval_config.seconds_per_visit)
             case _:
                 raise AttributeError('Not implemented')
 
+    def open(self, url: str) -> None:
+        args = self._get_terminal_args()
+        args.append(url)
+        self.process = TerminalAutomation.open_browser(args)
+
+    def terminate(self):
+        if self.process is None:
+            return
+
+        TerminalAutomation.terminate_browser(self.process, self._get_terminal_args())
+        self.process = None
+
     def pre_evaluation_setup(self):
         self.__fetch_binary()
 
@@ -46,11 +63,16 @@ def post_evaluation_cleanup(self):
 
     def pre_test_setup(self):
         self.__prepare_execution_folder()
-        self._prepare_profile_folder()
 
     def post_test_cleanup(self):
         self.__remove_execution_folder()
+
+    def pre_try_setup(self):
+        self._prepare_profile_folder()
+
+    def post_try_cleanup(self):
         self.__remove_profile_folder()
+        self.__empty_downloads_folder()
 
     def __fetch_binary(self):
         self.binary.fetch_binary()
@@ -70,21 +92,33 @@ def __remove_execution_folder(self):
         util.rmtree(self.__get_execution_folder_path())
 
     def __remove_profile_folder(self):
+        if self._profile_path is None:
+            return
         remove_profile_execution_folder(self._profile_path)
         self._profile_path = None
 
+    def __empty_downloads_folder(self):
+        download_folder = '/root/Downloads'
+        util.remove_all_in_folder(download_folder)
+
     def __get_execution_folder_path(self) -> str:
         return os.path.join(EXECUTION_PARENT_FOLDER, str(self.state.name))
 
     def _get_executable_file_path(self) -> str:
         return os.path.join(self.__get_execution_folder_path(), self.binary.executable_name)
 
     @abstractmethod
-    def _get_terminal_args(self):
+    def _get_terminal_args(self) -> list[str]:
+        pass
+
+    @abstractmethod
+    def get_navigation_sleep_duration(self) -> int:
         pass
 
     @staticmethod
-    def get_browser(browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, state: State) -> Browser:
+    def get_browser(
+        browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, state: State
+    ) -> Browser:
         from bci.browser.configuration.chromium import Chromium
         from bci.browser.configuration.firefox import Firefox
 

bci/browser/configuration/chromium.py

@@ -32,6 +32,9 @@
 
 class Chromium(Browser):
 
+    def get_navigation_sleep_duration(self) -> int:
+        return 1
+
     def _get_terminal_args(self) -> list[str]:
         assert self._profile_path is not None
 

bci/browser/configuration/firefox.py

@@ -2,10 +2,9 @@
 
 from bci import cli
 from bci.browser.configuration.browser import Browser
-from bci.browser.configuration.options import Default, BlockThirdPartyCookies, PrivateBrowsing, TrackingProtection
+from bci.browser.configuration.options import BlockThirdPartyCookies, Default, PrivateBrowsing, TrackingProtection
 from bci.browser.configuration.profile import prepare_firefox_profile
 
-
 SUPPORTED_OPTIONS = [
     Default(),
     BlockThirdPartyCookies(),
@@ -21,11 +20,15 @@
 
 class Firefox(Browser):
 
+    def get_navigation_sleep_duration(self) -> int:
+        return 2
+
     def _get_terminal_args(self) -> list[str]:
         assert self._profile_path is not None
 
         args = [self._get_executable_file_path()]
         args.extend(['-profile', self._profile_path])
+        args.append('-setDefaultBrowser')
         user_prefs = []
 
         def add_user_pref(key: str, value: str | int | bool):
@@ -45,6 +48,7 @@ def add_user_pref(key: str, value: str | int | bool):
         # add_user_pref('network.proxy.type', 1)
 
         add_user_pref('app.update.enabled', False)
+        add_user_pref('browser.shell.checkDefaultBrowser', False)
         if 'default' in self.browser_config.browser_setting:
             pass
         elif 'btpc' in self.browser_config.browser_setting:
@@ -92,7 +96,7 @@ def _prepare_profile_folder(self):
         if 'tp' in self.browser_config.browser_setting:
             self._profile_path = prepare_firefox_profile('tp-67')
         else:
-            self._profile_path = prepare_firefox_profile('default-67')
+            self._profile_path = prepare_firefox_profile()
 
         # Make Firefox trust the bughog CA
 

bci/browser/configuration/profile.py

@@ -1,34 +1,40 @@
 import os
+from typing import Optional
 
 from bci import cli
 
 PROFILE_STORAGE_FOLDER = '/app/browser/profiles'
 PROFILE_EXECUTION_FOLDER = '/tmp/profiles'
 
 
-def prepare_chromium_profile(profile_name: str = None) -> str:
+def prepare_chromium_profile(profile_name: Optional[str] = None) -> str:
     # Create new execution profile folder
     profile_execution_path = os.path.join(PROFILE_EXECUTION_FOLDER, 'new_profile')
     profile_execution_path = __create_folder(profile_execution_path)
 
     # Copy profile from storage to execution folder if profile_name is given
     if profile_name:
         if not os.path.exists(os.path.join(PROFILE_STORAGE_FOLDER, 'chromium', profile_name)):
-            raise Exception(f'Profile \'{profile_name}\' does not exist')
+            raise Exception(f"Profile '{profile_name}' does not exist")
         profile_storage_path = os.path.join(PROFILE_STORAGE_FOLDER, profile_name, 'Default')
         cli.execute(f'cp -r {profile_storage_path} {profile_execution_path}')
     return profile_execution_path
 
 
-def prepare_firefox_profile(profile_name: str = None) -> str:
+def prepare_firefox_profile(profile_name: Optional[str] = None) -> str:
+    """"
+    Create a temporary profile folder, based on the provided name of the premade profile.
+
+    :param profile_name: Reference to the premade profile folder used for creating the temporary profile.
+    """
     # Create new execution profile folder
     profile_execution_path = os.path.join(PROFILE_EXECUTION_FOLDER, 'new_profile')
     profile_execution_path = __create_folder(profile_execution_path)
 
     # Copy profile from storage to execution folder if profile_name is given
-    if profile_name is None:
+    if profile_name is not None:
         if not os.path.exists(os.path.join(PROFILE_STORAGE_FOLDER, 'firefox', profile_name)):
-            raise Exception(f'Profile \'{profile_name}\' does not exist')
+            raise Exception(f"Profile '{profile_name}' does not exist")
         profile_storage_path = os.path.join(PROFILE_STORAGE_FOLDER, profile_name)
         cli.execute(f'cp -r {profile_storage_path} {profile_execution_path}')
     return profile_execution_path