add factory test

tests/attacks/test_factory.py

@@ -0,0 +1,50 @@
+"""Test attack factory."""
+
+from __future__ import annotations
+
+import json
+import os
+
+import pytest
+import yaml
+from sklearn.ensemble import RandomForestClassifier
+
+from aisdc.attacks.factory import run_attacks
+from aisdc.config.attack import _get_attack
+
+
+@pytest.mark.parametrize(
+    "get_target", [RandomForestClassifier(random_state=1)], indirect=True
+)
+def test_factory(monkeypatch, get_target):
+    """Test Target object creation, saving, and loading."""
+    # create target_dir
+    target = get_target
+    target.save("target")
+
+    # create LiRA config with default params
+    mock_input = "yes"
+    monkeypatch.setattr("builtins.input", lambda _: mock_input)
+    attacks = []
+    attacks.append(_get_attack("lira"))
+
+    # create attack.yaml
+    filename: str = "attack.yaml"
+    with open(filename, "w", encoding="utf-8") as fp:
+        yaml.dump({"attacks": attacks}, fp)
+
+    # run attacks
+    run_attacks("target", "attack.yaml")
+
+    # load JSON report
+    path = os.path.normpath("outputs/report.json")
+    with open(path, encoding="utf-8") as fp:
+        report = json.load(fp)
+
+    # check report output
+    nr = list(report.keys())[0]
+    metrics = report[nr]["attack_experiment_logger"]["attack_instance_logger"][
+        "instance_0"
+    ]
+    tpr = metrics["TPR"]
+    assert tpr == pytest.approx(0.92, 0.01)

tests/conftest.py

@@ -14,6 +14,8 @@
 
 from aisdc.attacks.target import Target
 
+np.random.seed(1)
+
 folders = [
     "RES",
     "dt.sav",
@@ -31,6 +33,7 @@
     "release_dir",
     "safekeras.tf",
     "save_test",
+    "target",
     "test_lira_target",
     "test_output_lira",
     "test_output_sa",
@@ -89,7 +92,7 @@ def _cleanup():
 
 @pytest.fixture()
 def get_target(request) -> Target:  # pylint: disable=too-many-locals
-    """Wrap the model and data in a Target object.
+    """Return a target object with test data and fitted model.
 
     Uses a randomly sampled 10+10% of the nursery data set.
     """
@@ -123,13 +126,7 @@ def get_target(request) -> Target:  # pylint: disable=too-many-locals
         X_test_orig,
         y_train_orig,
         y_test_orig,
-    ) = train_test_split(
-        x,
-        y,
-        test_size=0.05,
-        stratify=y,
-        shuffle=True,
-    )
+    ) = train_test_split(x, y, test_size=0.05, stratify=y, shuffle=True, random_state=1)
 
     # now resample the training data reduce number of examples
     _, X_train_orig, _, y_train_orig = train_test_split(
@@ -138,6 +135,7 @@ def get_target(request) -> Target:  # pylint: disable=too-many-locals
         test_size=0.05,
         stratify=y_train_orig,
         shuffle=True,
+        random_state=1,
     )
 
     # [Researcher] Preprocess dataset
@@ -162,6 +160,9 @@ def get_target(request) -> Target:  # pylint: disable=too-many-locals
     xmore = np.concatenate((X_train_orig, X_test_orig))
     n_features = np.shape(X_train_orig)[1]
 
+    # fit model
+    model.fit(X_train, y_train)
+
     # wrap
     target = Target(model=model)
     target.dataset_name = "nursery"