Update testing matrix, start using image comparison (mdolab#40)

* Update testing matrix, start using image comparison * Update checkout action * Update reference images * Handle scientific notation output from odiff * Use images generated by github runner as ref * Increase image diff tolerance * Patch version bump
eirikurj · Feb 14, 2024 · b23b4d8 · b23b4d8
1 parent 7892f4c
commit b23b4d8
Show file tree

Hide file tree

Showing 19 changed files with 24 additions and 18 deletions.
diff --git a/.github/workflows/niceplots.yml b/.github/workflows/niceplots.yml
@@ -19,17 +19,12 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-        numpy-version: ["1.19.*", "1.21.*", "1.24.*"]
-        mpl-version: ["3.4.*", "3.6.*"]
-        exclude:
-          - python-version: "3.9"
-            numpy-version: "1.19.*"
-          - python-version: "3.10"
-            numpy-version: "1.19.*"
+      matrix: # Test 3 versions of python, and the latest and oldest supported versions of numpy and matplotlib, according to https://scientific-python.org/specs/spec-0000/
+        python-version: ["3.9", "3.10", "3.11"]
+        numpy-version: ["1.23.*", "1.26.*"]
+        mpl-version: ["3.6.*", "3.8.*"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -61,13 +56,13 @@ jobs:
           cd examples
           bash testExamples.sh
       - name: Compare against reference images
-        if: ${{ success() && matrix.python-version == '3.10' && matrix.numpy-version == '1.24.*' && matrix.mpl-version == '3.6.*' }}
+        if: ${{ success() && matrix.python-version == '3.11' && matrix.numpy-version == '1.26.*' && matrix.mpl-version == '3.8.*' }}
         run: |
           cd examples
           bash ImageComparisonTest.sh
       - name: Upload examples if failed
         uses: actions/upload-artifact@v3
-        if: ${{ failure() && matrix.python-version == '3.10' && matrix.numpy-version == '1.24.*' && matrix.mpl-version == '3.6.*' }}
+        if: ${{ failure() && matrix.python-version == '3.11' && matrix.numpy-version == '1.26.*' && matrix.mpl-version == '3.8.*' }}
         with:
           name: Examples
           path: examples/

diff --git a/examples/ImageComparisonTest.sh b/examples/ImageComparisonTest.sh
@@ -2,7 +2,8 @@
 
 # This script compares the images generated by the example scripts to a set of reference images
 
-test_passed=true # Kepp track of whether any tests fail
+test_passed=true # Keep track of whether any tests fail
+tolerance=0.05
 
 # First check that there aren't any images produced by the examples that don't have a reference image
 for f in *.png
@@ -25,9 +26,19 @@ do
     if test -f "$base_name"; then
         filename_no_ext="${base_name%.*}"
         echo "Comparing $base_name and $f"
-        odiff --antialiasing --threshold=0.1 --diff-mask $base_name $f diffs/${filename_no_ext}-diff.png
-        if [ $? -ne 0 ]; then
-            test_passed=false
+        output=$(odiff --parsable-stdout --antialiasing --diff-mask $base_name $f diffs/${filename_no_ext}-diff.png)
+        # echo "$output"
+        if [[ -z "$output" ]]; then
+            echo "Images match"
+        else
+            outputArray=(${output//;/ }) # Split the output by the semicolon
+            diffPixels=${outputArray[0]}
+            diffPercentage=$(printf "%.8f\n" ${outputArray[1]}) # use printf to convert scientific notation to decimal
+            echo "Difference = $diffPixels pixels ($diffPercentage %)"
+            if (( $(echo "$diffPercentage > $tolerance" | bc -l) )); then
+                echo "Difference is greater than $tolerance % so test failed"
+                test_passed=false
+            fi
         fi
     else
         echo "Couldn't find image to compare against $f"
@@ -44,5 +55,5 @@ if [ "$test_passed" = true ] ; then
 else
     echo "Some comparisons failed"
     echo "==========================="
-    exit 0 # In future (if we can make the image comparison more robust) we should exit with a non-zero exit code (e.g. 1)
+    exit 1
 fi
diff --git a/examples/ref/bar_chart.png b/examples/ref/bar_chart.png
diff --git a/examples/ref/colored_line.png b/examples/ref/colored_line.png
diff --git a/examples/ref/colored_line_custom_norm.png b/examples/ref/colored_line_custom_norm.png
diff --git a/examples/ref/default_style_demo.png b/examples/ref/default_style_demo.png
diff --git a/examples/ref/doumont-dark_style_demo.png b/examples/ref/doumont-dark_style_demo.png
diff --git a/examples/ref/doumont-light_style_demo.png b/examples/ref/doumont-light_style_demo.png
diff --git a/examples/ref/james-dark_style_demo.png b/examples/ref/james-dark_style_demo.png
diff --git a/examples/ref/james-light_style_demo.png b/examples/ref/james-light_style_demo.png
diff --git a/examples/ref/line_end_labels.png b/examples/ref/line_end_labels.png
diff --git a/examples/ref/nested_pie_chart.png b/examples/ref/nested_pie_chart.png
diff --git a/examples/ref/opt_prob-hashed.png b/examples/ref/opt_prob-hashed.png
diff --git a/examples/ref/opt_prob-shaded.png b/examples/ref/opt_prob-shaded.png
diff --git a/examples/ref/opt_stacks.png b/examples/ref/opt_stacks.png
diff --git a/examples/ref/opt_stacks_more_data.png b/examples/ref/opt_stacks_more_data.png
diff --git a/examples/ref/parula_contours.png b/examples/ref/parula_contours.png
diff --git a/examples/ref/style_color_demo.png b/examples/ref/style_color_demo.png
diff --git a/niceplots/__init__.py b/niceplots/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.5.0"
+__version__ = "2.5.1"
 
 from .utils import *
 from .parula import *