diff --git a/test/test_Phylo.jl b/test/test_Phylo.jl
index ecbc6be8..d80e5fb0 100644
--- a/test/test_Phylo.jl
+++ b/test/test_Phylo.jl
@@ -7,17 +7,25 @@ using Git
 using Logging
 using Pkg
 
-function is_repo_clean(repo_path)
+function is_repo_clean(repo_path; ignore_untracked = true)
     # Get the status of the repository
     statuses = readlines(`$(Git.git()) status -s $repo_path`)
-    statuses = filter((!) ∘ contains("??"), statuses)
+
+    if ignore_untracked
+        # Repo must be clean except for untracked files
+        statuses = filter((!) ∘ contains("??"), statuses)
+    end
 
     is_clean = isempty(statuses)
+
+    # If not clean then report on dirty files
     is_clean || @error "\n" * join(statuses, "\n")
 
     return is_clean
 end
 
+# Metadata crosswalk testing only works on Julia v1.8 and after due to Project.toml changes
+# Also does not currently work on Windows runners on GitHub due to file writing issues
 if VERSION ≥ VersionNumber("1.8.0") && (!haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows")
     Pkg.develop(url = "https://github.com/richardreeve/ResearchSoftwareMetadata.jl.git")
     using ResearchSoftwareMetadata