Adding the Better Tracing Suite into Main

.git-blame-ignore-revs

@@ -1,3 +1,7 @@
+# nodestats codespell corrections
+f13f8d40d6768f14a46315d105b3d80beaa33aaf
+284f113cf8416eed43b88b4c438d7552456616e3
+
 # numpydoc validation - Linting docstrings with pre-commit numpydoc
 836e1bf5347eb7a9f97e784e783045c0287b3fe9
 2580811edefac867938ee0c4b705649a493e5d4f

.github/workflows/tests.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

.gitignore

@@ -23,6 +23,7 @@ output/config.yaml
 *.npy
 *.csv
 *.spm
+*.svg
 
 # Notebook checkpoints
 notebooks/.ipynb_checkpoints/
@@ -34,6 +35,7 @@ notebooks/.ipynb_checkpoints/
 .vscode/
 .idea/
 *~
+pytest-debug.ini
 
 # Documentation
 _build/
@@ -57,5 +59,10 @@ topostats/_version.py
 # default output directory, often common from testing
 output/
 
+# Include all files in tests and all subdirectories except processed and __pycache__
+!tests/**
+tests/resources/processed/
+__pycache__/
+
 # Debugging
-pytest-debug.ini
+pytest-debug.ini

.markdownlint-cli2.yaml

@@ -8,6 +8,7 @@ config:
   html:
     allowed_elements:
       - div
+      - br
 
 # Globs
 globs:

pyproject.toml

@@ -34,7 +34,7 @@ keywords = [
   "afm",
   "image processing"
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.9, <3.12"
 dependencies = [
   "AFMReader",
   "h5py",
@@ -44,18 +44,20 @@ dependencies = [
   "numpy",
   "numpyencoder",
   "pandas",
-  "pySPM",
   "pyfiglet",
+  "pySPM",
   "pyyaml",
   "ruamel.yaml",
   "schema",
   "scikit-image",
   "scipy",
   "seaborn",
+  "skan",
   "snoop",
+  "tensorflow",
   "tifffile",
+  "topoly",
   "tqdm",
-  "tensorflow",
 ]
 
 [project.optional-dependencies]
@@ -127,8 +129,10 @@ write_to = "topostats/_version.py"
 
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = ["--mpl", "-ra", "--strict-config", "--strict-markers"]
-log_cli_level = "Info"
+addopts = ["--cov", "--mpl", "-ra", "--strict-config", "--strict-markers"]
+log_level = "INFO"
+log_cli = true
+log_cli_level = "INFO"
 testpaths = [
     "tests",
 ]
@@ -185,7 +189,6 @@ exclude = [
   "dist",
   "docs/conf.py",
   "node_modules",
-  "pygwytracing.py",
   "tests/tracing/test_dnacurvature.py",
   "tests/tracing/test_dnatracing.py",
   "tests/tracing/test_tracing_dna.py",
@@ -258,7 +261,7 @@ convention = "numpy"
 fixture-parentheses = true
 
 [tool.codespell]
-skip = '*.spm*,*.mplstyle'
+skip = '*.spm*,*.mplstyle,*.svg'
 count = ''
 quiet-level = 3
 
@@ -275,9 +278,10 @@ exclude = [  # don't report on objects that match any of these regex
     "\\.__repr__$",
     "^test_",
     "^conftest",
-    "^dnatrcing",
+    "^nodestats",
     "^tracingfuncs",
     "^conf$",
+    "^theme",
 ]
 override_SS05 = [  # override SS05 to allow docstrings starting with these words
     "^Process ",

temp.ipynb

@@ -0,0 +1,103 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from scipy.spatial.distance import cdist"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# point arrays\n",
+    "array1 = np.array([[1, 2], [2, 3], [3, 3]])\n",
+    "array2 = np.array([[4, 4], [4, 5], [5, 5], [6, 6]])\n",
+    "\n",
+    "img = np.zeros((10, 10))\n",
+    "# set all array1 points to 1\n",
+    "for point in array1:\n",
+    "    print(point)\n",
+    "    img[point[0], point[1]] = 1\n",
+    "\n",
+    "print(\"----\")\n",
+    "\n",
+    "# set all array2 points to 2\n",
+    "for point in array2:\n",
+    "    print(point)\n",
+    "    img[point[0], point[1]] = 2\n",
+    "\n",
+    "plt.imshow(img)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Numpy approach\n",
+    "\n",
+    "# How to create this matrix from array1 and array2?\n",
+    "# [1 4] [1 4] [1 5] [1 6]\n",
+    "# [2 4] [2 4] [2 5] [2 6]\n",
+    "# [3 4] [3 4] [3 5] [3 6]\n",
+    "\n",
+    "grid1, grid2 = np.meshgrid(array1, array2)\n",
+    "print(grid1)\n",
+    "print(grid2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test if any points are touching between the two arrays, using vectorisation\n",
+    "\n",
+    "diffs = np.subtract.outer(array1, array2)\n",
+    "print(diffs)\n",
+    "print(\"---\")\n",
+    "# linalg the diffs\n",
+    "dists = np.linalg.norm(diffs, axis=1)\n",
+    "print(dists)\n",
+    "\n",
+    "print(\"---\")\n",
+    "\n",
+    "\n",
+    "cdists = cdist(array1, array2, \"euclidean\")\n",
+    "print(cdists)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "new",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/_regtest_outputs/test_grainstats_minicircle.test_grainstats_regression.out

@@ -1,5 +1,5 @@
                      centre_x      centre_y    radius_min    radius_max   radius_mean  radius_median    height_min    height_max  height_median   height_mean        volume          area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold     max_feret     min_feret image
-molecule_number                                                                                                                                                                                                                                                                                                                                    
+grain_number                                                                                                                                                                                                                                                                                                                                    
 0                7.500895e-08  4.775362e-08  4.044948e-09  2.542964e-08  1.600048e-08   1.647888e-08  1.006306e-09  2.696640e-09   1.597337e-09  1.596062e-09  1.084371e-24  6.794043e-16         1.319762e-15             2.053897e-08              5.037861e-08            1.034725e-15      0.407692     above  5.037861e-08  2.053897e-08  None
 1                8.028649e-08  7.895237e-08  6.658610e-09  2.623645e-08  1.609777e-08   1.573693e-08  1.009513e-09  2.480685e-09   1.668932e-09  1.683197e-09  1.031843e-24  6.130257e-16         1.526707e-15             2.017441e-08              4.942135e-08            9.970467e-16      0.408213     above  4.987228e-08  2.017441e-08  None
 2                4.001424e-08  7.585689e-08  9.745734e-09  2.369314e-08  1.746942e-08   1.818722e-08  1.006681e-09  2.139365e-09   1.646541e-09  1.598842e-09  1.123717e-24  7.028320e-16         1.546230e-15             3.359220e-08              4.149625e-08            1.393950e-15      0.809524     above  4.440534e-08  3.176819e-08  None

tests/_regtest_outputs/test_processing.test_process_scan_above.out

@@ -1,8 +1,7 @@
                   image_size_x_m  image_size_y_m  image_area_m2  image_size_x_px  image_size_y_px  image_area_px2  grains_number_above  grains_per_m2_above  grains_number_below  grains_per_m2_below  rms_roughness
 image                                                                                                                                                                                                               
 minicircle_small      1.2646e-07      1.2646e-07     1.5993e-14               64               64            4096                    3           1.8758e+14                    0           0.0000e+00     6.8208e-10
-                  centre_x   centre_y  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean     volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image  contour_length  circular  end_to_end_distance
-molecule_number                                                                                                                                                                                                                                                                                                                                                                   
-0               7.5100e-08 4.7559e-08  3.9431e-09  2.5631e-08   1.6016e-08     1.6680e-08  9.1991e-10  2.6422e-09     1.5338e-09   1.5341e-09 1.0543e-24 6.8721e-16           1.3198e-15               2.0539e-08                5.0379e-08              1.0347e-15    4.0769e-01     above 5.0379e-08 2.0539e-08  minicircle_small      6.0226e-08     False           8.6738e-09
-1               8.0241e-08 7.8677e-08  6.8951e-09  2.7188e-08   1.6272e-08     1.6263e-08  9.0630e-10  2.4586e-09     1.6144e-09   1.6264e-09 1.0352e-24 6.3645e-16           1.5931e-15               2.0174e-08                5.1212e-08              1.0332e-15    3.9394e-01     above 5.1262e-08 2.0174e-08  minicircle_small      6.6355e-08      True           0.0000e+00
-2               4.0012e-08 7.5644e-08  9.9461e-09  2.3654e-08   1.7561e-08     1.8364e-08  9.0641e-10  2.1066e-09     1.5939e-09   1.5493e-09 1.1192e-24 7.2236e-16           1.5462e-15               3.3592e-08                4.1496e-08              1.3940e-15    8.0952e-01     above 4.4405e-08 3.2528e-08  minicircle_small      9.6106e-08      True           0.0000e+00
+    centre_x   centre_y  grain_number  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean     volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image  grain_endpoints  grain_junctions  total_branch_lengths  num_crossings avg_crossing_confidence min_crossing_confidence  num_mols writhe_string  total_contour_length  average_end_to_end_distance
+0 7.5100e-08 4.7559e-08             0  3.9431e-09  2.5631e-08   1.6016e-08     1.6680e-08  9.1991e-10  2.6422e-09     1.5338e-09   1.5341e-09 1.0543e-24 6.8721e-16           1.3198e-15               2.0539e-08                5.0379e-08              1.0347e-15    4.0769e-01     above 5.0379e-08 2.0539e-08  minicircle_small                1                1            8.4571e+01              1                    None                    None         2                          6.5881e+01                   8.8370e+00
+1 8.0241e-08 7.8677e-08             1  6.8951e-09  2.7188e-08   1.6272e-08     1.6263e-08  9.0630e-10  2.4586e-09     1.6144e-09   1.6264e-09 1.0352e-24 6.3645e-16           1.5931e-15               2.0174e-08                5.1212e-08              1.0332e-15    3.9394e-01     above 5.1262e-08 2.0174e-08  minicircle_small                0                0            7.3054e+01              0                    None                    None         1           NaN            5.8272e+01                   0.0000e+00
+2 4.0012e-08 7.5644e-08             2  9.9461e-09  2.3654e-08   1.7561e-08     1.8364e-08  9.0641e-10  2.1066e-09     1.5939e-09   1.5493e-09 1.1192e-24 7.2236e-16           1.5462e-15               3.3592e-08                4.1496e-08              1.3940e-15    8.0952e-01     above 4.4405e-08 3.2528e-08  minicircle_small                0                0            1.0447e+02              0                    None                    None         1           NaN            8.7183e+01                   0.0000e+00

tests/conftest.py

@@ -141,6 +141,8 @@ def grainstats_config(default_config: dict) -> dict:
     """Configurations for grainstats."""
     config = default_config["grainstats"]
     config["direction"] = "above"
+    # Set cropped image size to 40nm
+    config["cropped_size"] = 40.0
     config.pop("run")
     return config
 
@@ -732,30 +734,39 @@ def minicircle_grainstats(
     )
 
 
-# Derive fixtures for DNA Tracing
-GRAINS = np.array(
-    [
-        [0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 2],
-        [0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2],
-        [0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2],
-        [0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 2],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2],
-        [0, 0, 3, 3, 3, 3, 3, 0, 0, 0, 2],
-        [0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 2],
-        [0, 0, 3, 3, 3, 3, 3, 0, 0, 0, 2],
-        [0, 0, 4, 4, 4, 4, 4, 0, 0, 0, 2],
-    ]
-)
-FULL_IMAGE = RNG.random((GRAINS.shape[0], GRAINS.shape[1]))
+# Random shapes
+# Generate a random skeletons, first is a skeleton with a closed loop with side branches
+kwargs = {
+    "image_shape": (60, 32),
+    "max_shapes": 10,
+    "channel_axis": None,
+    "shape": None,
+    "allow_overlap": True,
+    "min_size": 20,
+}
 
 
 @pytest.fixture()
-def test_dnatracing() -> dnaTrace:
-    """Instantiate a dnaTrace object."""
-    return dnaTrace(image=FULL_IMAGE, grain=GRAINS, filename="Test", pixel_to_nm_scaling=1.0)
+def utils_skeleton_linear1() -> npt.NDArray:
+    """Linear skeleton."""
+    random_images, _ = draw.random_shapes(rng=1, **kwargs)
+    return skeletonize(random_images != 255)
 
 
 @pytest.fixture()
+def utils_skeleton_linear2() -> npt.NDArray:
+    """Linear skeleton T-junction and side-branch."""
+    random_images, _ = draw.random_shapes(rng=165103, **kwargs)
+    return skeletonize(random_images != 255)
+
+
+@pytest.fixture()
+def utils_skeleton_linear3() -> npt.NDArray:
+    """Linear skeleton with several branches."""
+    random_images, _ = draw.random_shapes(rng=7334281, **kwargs)
+    return skeletonize(random_images != 255)
+
+
 def minicircle_dnatracing(
     minicircle_grain_gaussian_filter: Filters,
     minicircle_grain_coloured: Grains,
@@ -900,11 +911,13 @@ def _generate_random_skeleton(**extra_kwargs):
         "shape": None,
         "allow_overlap": True,
     }
+    # kwargs.update
     heights = {"scale": 100, "sigma": 5.0, "cval": 20.0}
-    random_image, _ = draw.random_shapes(**kwargs, **extra_kwargs)
+    kwargs = {**kwargs, **extra_kwargs}
+    random_image, _ = draw.random_shapes(**kwargs)
     mask = random_image != 255
     skeleton = skeletonize(mask)
-    return {"img": _generate_heights(skeleton, **heights), "skeleton": skeleton}
+    return {"original": mask, "img": _generate_heights(skeleton, **heights), "skeleton": skeleton}
 
 
 @pytest.fixture()
@@ -937,13 +950,21 @@ def skeleton_linear3() -> dict:
     return _generate_random_skeleton(rng=894632511, min_size=20)
 
 
-# Helper functions for visualising skeletons and heights
-#
+@pytest.fixture()
+def pruning_skeleton() -> dict:
+    """Smaller skeleton for testing parameters of prune_all_skeletons(). Has a T-junction."""
+    return _generate_random_skeleton(rng=69432138, min_size=15, image_shape=(30, 30))
+
+
+## Helper function visualising for generating skeletons and heights
+
+
+# import matplotlib.pyplot as plt
 # def pruned_plot(gen_shape: dict) -> None:
 #     """Plot the original skeleton, its derived height and the pruned skeleton."""
-#     img_skeleton = gen_shape()
+#     img_skeleton = gen_shape
 #     pruned = topostatsPrune(
-#         img_skeleton["heights"],
+#         img_skeleton["img"],
 #         img_skeleton["skeleton"],
 #         max_length=-1,
 #         height_threshold=90,
@@ -952,17 +973,23 @@ def skeleton_linear3() -> dict:
 #     )
 #     pruned_skeleton = pruned._prune_by_length(pruned.skeleton, pruned.max_length)
 #     fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2)
-#     ax1.imshow(img_skeleton["skeleton"])
-#     ax2.imshow(img_skeleton["heights"])
-#     ax3.imshow(pruned_skeleton)
+#     ax1.imshow(img_skeleton["original"])
+#     ax1.set_title("Original mask")
+#     ax2.imshow(img_skeleton["skeleton"])
+#     ax2.set_title("Skeleton")
+#     ax3.imshow(img_skeleton["img"])
+#     ax3.set_title("Gaussian Blurring")
+#     ax4.imshow(pruned_skeleton)
+#     ax4.set_title("Pruned Skeleton")
 #     plt.show()
 
 
-# pruned_plot(skeleton_loop1)
-# pruned_plot(skeleton_loop2)
-# pruned_plot(skeleton_linear1)
-# pruned_plot(skeleton_linear2)
-# pruned_plot(skeleton_linear3)
+# pruned_plot(pruning_skeleton_loop1())
+# pruned_plot(pruning_skeleton_loop2())
+# pruned_plot(pruning_skeleton_linear1())
+# pruned_plot(pruning_skeleton_linear2())
+# pruned_plot(pruning_skeleton_linear3())
+# pruned_plot(pruning_skeleton())
 
 
 # U-Net fixtures