From 51da19b6ea2c686951273886ee83fb1d97aa36b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:10:32 -0500 Subject: [PATCH 01/20] Build(deps): Bump codecov/codecov-action from 3 to 4 (#412) * Build(deps): Bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * CI: Pass token to codecov action --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Chris Markiewicz --- .github/workflows/pythonpackage.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d95b7c71f6..daf66fe141 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -109,8 +109,10 @@ jobs: run: python -m pip install "smriprep[tests]" - name: Run tests run: pytest -sv --doctest-modules --cov smriprep --pyargs smriprep - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 name: Submit to CodeCov + with: + token: ${{ secrets.CODECOV_TOKEN }} style: if: "!contains(github.event.head_commit.message, '[skip ci]')" From 2355035d18d886cef5b6f0f4771e00c607d897a1 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 28 Feb 2024 13:47:47 -0500 Subject: [PATCH 02/20] FIX: Re-add cohort identifier to template name (#416) * FIX: Re-add cohort identifier to template name * FIX: XFM io * FIX: Query data with nipreps config Also rename variables to play nice with pdb * TST: Add test for derivatives ingestion * PIN: niworkflows latest * MAINT: Update ruff settings * STY: ruff fixes * TST: Ensure config is string * Apply suggestions from code review Co-authored-by: Chris Markiewicz --------- Co-authored-by: Chris Markiewicz --- pyproject.toml | 8 +- smriprep/data/io_spec.json | 8 +- smriprep/utils/bids.py | 37 ++++----- smriprep/utils/tests/__init__.py | 5 ++ smriprep/utils/tests/derivatives.yml | 101 +++++++++++++++++++++++++ smriprep/utils/tests/test_bids.py | 30 ++++++++ smriprep/workflows/fit/registration.py | 19 ++++- 7 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 smriprep/utils/tests/derivatives.yml create mode 100644 smriprep/utils/tests/test_bids.py diff --git a/pyproject.toml b/pyproject.toml index f57a362916..138234a55b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "matplotlib >= 2.2.0", "nibabel >= 4.0.1", "nipype >= 1.7.0", - "niworkflows >= 1.8.0", + "niworkflows @ git+https://github.com/nipreps/niworkflows.git@master", "numpy", "packaging", "pybids >= 0.11.1", @@ -138,6 +138,8 @@ source = [ [tool.ruff] line-length = 99 + +[tool.ruff.lint] extend-select = [ "F", "E", @@ -162,10 +164,10 @@ extend-select = [ "Q", ] -[tool.ruff.flake8-quotes] +[tool.ruff.lint.flake8-quotes] inline-quotes = "single" -[tool.ruff.extend-per-file-ignores] +[tool.ruff.lint.extend-per-file-ignores] "*/test_*.py" = ["S101"] [tool.ruff.format] diff --git a/smriprep/data/io_spec.json b/smriprep/data/io_spec.json index 6b8fcd50fc..54edaf3ac1 100644 --- a/smriprep/data/io_spec.json +++ b/smriprep/data/io_spec.json @@ -50,8 +50,8 @@ "forward": { "datatype": "anat", "extension": [ - "h5", - "txt" + ".h5", + ".txt" ], "from": "T1w", "to": null, @@ -61,8 +61,8 @@ "reverse": { "datatype": "anat", "extension": [ - "h5", - "txt" + ".h5", + ".txt" ], "from": null, "to": "T1w", diff --git a/smriprep/utils/bids.py b/smriprep/utils/bids.py index 123f8dd6c1..c213a0afc3 100644 --- a/smriprep/utils/bids.py +++ b/smriprep/utils/bids.py @@ -25,6 +25,7 @@ from pathlib import Path from bids.layout import BIDSLayout +from niworkflows.data import load as nwf_load from ..data import load_resource @@ -39,36 +40,38 @@ def collect_derivatives(derivatives_dir, subject_id, std_spaces, spec=None, patt if patterns is None: patterns = _patterns - layout = BIDSLayout(derivatives_dir, config=['bids', 'derivatives'], validate=False) + deriv_config = nwf_load('nipreps.json') + layout = BIDSLayout(derivatives_dir, config=deriv_config, validate=False) derivs_cache = {} - for k, q in spec['baseline'].items(): - q['subject'] = subject_id - item = layout.get(return_type='filename', **q) + for key, qry in spec['baseline'].items(): + qry['subject'] = subject_id + item = layout.get(return_type='filename', **qry) if not item: continue - derivs_cache['t1w_%s' % k] = item[0] if len(item) == 1 else item + derivs_cache[f't1w_{key}'] = item[0] if len(item) == 1 else item transforms = derivs_cache.setdefault('transforms', {}) - for space in std_spaces: - for k, q in spec['transforms'].items(): - q = q.copy() - q['subject'] = subject_id - q['from'] = q['from'] or space - q['to'] = q['to'] or space - item = layout.get(return_type='filename', **q) + for _space in std_spaces: + space = _space.replace(':cohort-', '+') + for key, qry in spec['transforms'].items(): + qry = qry.copy() + qry['subject'] = subject_id + qry['from'] = qry['from'] or space + qry['to'] = qry['to'] or space + item = layout.get(return_type='filename', **qry) if not item: continue - transforms.setdefault(space, {})[k] = item[0] if len(item) == 1 else item + transforms.setdefault(_space, {})[key] = item[0] if len(item) == 1 else item - for k, q in spec['surfaces'].items(): - q['subject'] = subject_id - item = layout.get(return_type='filename', **q) + for key, qry in spec['surfaces'].items(): + qry['subject'] = subject_id + item = layout.get(return_type='filename', **qry) if not item or len(item) != 2: continue - derivs_cache[k] = sorted(item) + derivs_cache[key] = sorted(item) return derivs_cache diff --git a/smriprep/utils/tests/__init__.py b/smriprep/utils/tests/__init__.py index e69de29bb2..84a88300dc 100644 --- a/smriprep/utils/tests/__init__.py +++ b/smriprep/utils/tests/__init__.py @@ -0,0 +1,5 @@ +from niworkflows.data import Loader + +load_data = Loader(__package__) + +DERIV_SKELETON = load_data('derivatives.yml') diff --git a/smriprep/utils/tests/derivatives.yml b/smriprep/utils/tests/derivatives.yml new file mode 100644 index 0000000000..97cda9e19c --- /dev/null +++ b/smriprep/utils/tests/derivatives.yml @@ -0,0 +1,101 @@ +dataset_description: + Name: smriprep-outputs + BIDSVersion: 1.9.0 + DatasetType: derivative +'01': + anat: + - suffix: mask + desc: brain + - suffix: T1w + desc: preproc + - suffix: dseg + - suffix: probseg + label: CSF + - suffix: probseg + label: GM + - suffix: probseg + label: WM + - suffix: xfm + from: MNI152NLin2009cAsym + to: T1w + mode: image + extension: .h5 + - suffix: xfm + from: T1w + to: MNI152NLin2009cAsym + mode: image + extension: .h5 + - suffix: xfm + from: T1w + to: MNIPediatricAsym+3 + mode: image + extension: .h5 + - suffix: xfm + from: MNIPediatricAsym+3 + to: T1w + mode: image + extension: .h5 + - suffix: white + hemi: L + extension: .surf.gii + - suffix: white + hemi: R + extension: .surf.gii + - suffix: pial + hemi: L + extension: .surf.gii + - suffix: pial + hemi: R + extension: .surf.gii + - suffix: midthickness + hemi: L + extension: .surf.gii + - suffix: midthickness + hemi: R + extension: .surf.gii + - suffix: sphere + hemi: L + extension: .surf.gii + - suffix: sphere + hemi: R + extension: .surf.gii + - suffix: sphere + hemi: L + desc: reg + extension: .surf.gii + - suffix: sphere + hemi: R + desc: reg + extension: .surf.gii + - suffix: sphere + hemi: L + space: fsLR + desc: reg + extension: .surf.gii + - suffix: sphere + hemi: R + space: fsLR + desc: reg + extension: .surf.gii + - suffix: sphere + hemi: L + space: fsLR + desc: msmsulc + extension: .surf.gii + - suffix: sphere + hemi: R + space: fsLR + desc: msmsulc + extension: .surf.gii + - suffix: thickness + hemi: L + extension: .shape.gii + - suffix: thickness + hemi: R + extension: .shape.gii + - suffix: sulc + hemi: L + extension: .shape.gii + - suffix: sulc + hemi: R + extension: .shape.gii \ No newline at end of file diff --git a/smriprep/utils/tests/test_bids.py b/smriprep/utils/tests/test_bids.py new file mode 100644 index 0000000000..a8b094ea99 --- /dev/null +++ b/smriprep/utils/tests/test_bids.py @@ -0,0 +1,30 @@ +from niworkflows.utils.testing import generate_bids_skeleton + +from ..bids import collect_derivatives +from . import DERIV_SKELETON + + +def test_collect_derivatives(tmp_path): + deriv_dir = tmp_path / 'derivatives' + generate_bids_skeleton(deriv_dir, str(DERIV_SKELETON)) + output_spaces = ['MNI152NLin2009cAsym', 'MNIPediatricAsym:cohort-3'] + collected = collect_derivatives(deriv_dir, '01', output_spaces) + for suffix in ('preproc', 'mask', 'dseg'): + assert collected[f't1w_{suffix}'] + assert len(collected['t1w_tpms']) == 3 + xfms = collected['transforms'] + for space in output_spaces: + assert xfms[space]['reverse'] + assert xfms[space]['forward'] + for surface in ( + 'white', + 'pial', + 'midthickness', + 'sphere', + 'thickness', + 'sulc', + 'sphere_reg', + 'sphere_reg_fsLR', + 'sphere_reg_msm', + ): + assert len(collected[surface]) == 2 diff --git a/smriprep/workflows/fit/registration.py b/smriprep/workflows/fit/registration.py index 6e83212425..f9df9fb901 100644 --- a/smriprep/workflows/fit/registration.py +++ b/smriprep/workflows/fit/registration.py @@ -185,6 +185,12 @@ def init_register_template_wf( mem_gb=2, ) + fmt_cohort = pe.Node( + niu.Function(function=_fmt_cohort, output_names=['template', 'spec']), + name='fmt_cohort', + run_without_submitting=True, + ) + # fmt:off workflow.connect([ (inputnode, split_desc, [('template', 'template')]), @@ -202,8 +208,12 @@ def init_register_template_wf( ]), (trunc_mov, registration, [ ('output_image', 'moving_image')]), - (split_desc, outputnode, [ + (split_desc, fmt_cohort, [ ('name', 'template'), + ('spec', 'spec'), + ]), + (fmt_cohort, outputnode, [ + ('template', 'template'), ('spec', 'template_spec'), ]), (registration, outputnode, [ @@ -225,3 +235,10 @@ def _make_outputnode(workflow, out_fields, joinsource): workflow.connect([(pout, out, [(f, f) for f in out_fields])]) return pout return pe.Node(niu.IdentityInterface(fields=out_fields), name='outputnode') + + +def _fmt_cohort(template, spec): + cohort = spec.pop('cohort', None) + if cohort is not None: + template = f'{template}:cohort-{cohort}' + return template, spec From 8efea4b7d7b3c245bd5894b2f465a396525580c0 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 7 Mar 2024 14:02:44 -0500 Subject: [PATCH 03/20] CHORE: Update ruff, ignore certain rules --- .github/workflows/pythonpackage.yml | 2 +- .pre-commit-config.yaml | 2 +- pyproject.toml | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index daf66fe141..1d17c89d58 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -119,5 +119,5 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: pipx run ruff smriprep + - run: pipx run ruff check smriprep - run: pipx run ruff format --diff smriprep diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6cf50e215..ee84b53cca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: check-toml - id: check-json - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + rev: v0.2.0 hooks: - id: ruff - id: ruff-format diff --git a/pyproject.toml b/pyproject.toml index 138234a55b..0bd878e582 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,6 +163,11 @@ extend-select = [ "PT", "Q", ] +extend-ignore = [ + "S311", # We are not using random for cryptographic purposes + "ISC001", + "S603", +] [tool.ruff.lint.flake8-quotes] inline-quotes = "single" From fb65496153313ca65e4f242f2ea0640e66cc378d Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Thu, 7 Mar 2024 14:34:43 -0500 Subject: [PATCH 04/20] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ee84b53cca..c392fd2b85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: check-toml - id: check-json - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.3.1 hooks: - id: ruff - id: ruff-format From 83a50d465145c6e8176c3f13d4673ed4e0bdb26f Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 7 Mar 2024 14:40:13 -0500 Subject: [PATCH 05/20] STY: ruff format smriprep [git-blame-ignore-rev] --- smriprep/__init__.py | 1 + smriprep/conf/__init__.py | 1 + smriprep/interfaces/freesurfer.py | 1 + smriprep/interfaces/gifti.py | 1 + smriprep/interfaces/surf.py | 1 + smriprep/interfaces/templateflow.py | 1 + smriprep/utils/bids.py | 1 + smriprep/workflows/anatomical.py | 1 + smriprep/workflows/base.py | 1 + smriprep/workflows/fit/registration.py | 1 + smriprep/workflows/outputs.py | 1 + smriprep/workflows/surfaces.py | 1 + 12 files changed, 12 insertions(+) diff --git a/smriprep/__init__.py b/smriprep/__init__.py index 13c2714ab2..8f41228dcb 100644 --- a/smriprep/__init__.py +++ b/smriprep/__init__.py @@ -8,6 +8,7 @@ """ + from .__about__ import ( __copyright__, __credits__, diff --git a/smriprep/conf/__init__.py b/smriprep/conf/__init__.py index 97cbd76598..39e82ec06a 100644 --- a/smriprep/conf/__init__.py +++ b/smriprep/conf/__init__.py @@ -1,4 +1,5 @@ """sMRIPrep settings.""" + from templateflow.api import templates as _get_templates TF_TEMPLATES = tuple(_get_templates()) diff --git a/smriprep/interfaces/freesurfer.py b/smriprep/interfaces/freesurfer.py index bd44550f9f..94cd2960fa 100644 --- a/smriprep/interfaces/freesurfer.py +++ b/smriprep/interfaces/freesurfer.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Nipype's recon-all replacement.""" + import os from looseversion import LooseVersion diff --git a/smriprep/interfaces/gifti.py b/smriprep/interfaces/gifti.py index 08bc740bee..979902a191 100644 --- a/smriprep/interfaces/gifti.py +++ b/smriprep/interfaces/gifti.py @@ -1,4 +1,5 @@ """Interfaces for manipulating GIFTI files.""" + import os import nibabel as nb diff --git a/smriprep/interfaces/surf.py b/smriprep/interfaces/surf.py index 107c679ca3..4022020b6a 100644 --- a/smriprep/interfaces/surf.py +++ b/smriprep/interfaces/surf.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Handling surfaces.""" + import os from pathlib import Path diff --git a/smriprep/interfaces/templateflow.py b/smriprep/interfaces/templateflow.py index bb6374984c..e7199c8b94 100644 --- a/smriprep/interfaces/templateflow.py +++ b/smriprep/interfaces/templateflow.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Interfaces to get templates from TemplateFlow.""" + import logging from nipype.interfaces.base import ( diff --git a/smriprep/utils/bids.py b/smriprep/utils/bids.py index c213a0afc3..958cffa33b 100644 --- a/smriprep/utils/bids.py +++ b/smriprep/utils/bids.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Utilities to handle BIDS inputs.""" + from json import loads from pathlib import Path diff --git a/smriprep/workflows/anatomical.py b/smriprep/workflows/anatomical.py index 305f3ed08e..e65620da47 100644 --- a/smriprep/workflows/anatomical.py +++ b/smriprep/workflows/anatomical.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Anatomical reference preprocessing workflows.""" + import typing as ty from nipype import logging diff --git a/smriprep/workflows/base.py b/smriprep/workflows/base.py index a62ed135b1..5e1bda87e6 100644 --- a/smriprep/workflows/base.py +++ b/smriprep/workflows/base.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """*sMRIPrep* base processing workflows.""" + import os import sys from copy import deepcopy diff --git a/smriprep/workflows/fit/registration.py b/smriprep/workflows/fit/registration.py index f9df9fb901..6365b3c7bc 100644 --- a/smriprep/workflows/fit/registration.py +++ b/smriprep/workflows/fit/registration.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Spatial normalization workflows.""" + from collections import defaultdict from nipype.interfaces import ants diff --git a/smriprep/workflows/outputs.py b/smriprep/workflows/outputs.py index d7abe11f87..82f0f668c8 100644 --- a/smriprep/workflows/outputs.py +++ b/smriprep/workflows/outputs.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Writing outputs.""" + import typing as ty from nipype.interfaces import utility as niu diff --git a/smriprep/workflows/surfaces.py b/smriprep/workflows/surfaces.py index 62ad2ea94b..b32c2e2862 100644 --- a/smriprep/workflows/surfaces.py +++ b/smriprep/workflows/surfaces.py @@ -27,6 +27,7 @@ structural images. """ + import typing as ty from nipype.interfaces import freesurfer as fs From 7378ee1562589cf42fc6aa754129911142cedaa4 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 7 Mar 2024 14:41:52 -0500 Subject: [PATCH 06/20] MNT: Update .git-blame-ignore-revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 663a141052..988964e066 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,5 @@ +# 2024-03-07 - mathiasg@stanford.edu - STY: ruff format smriprep [git-blame-ignore-rev] +83a50d465145c6e8176c3f13d4673ed4e0bdb26f # 2023-11-21 - effigies@gmail.com - STY: ruff --fix smriprep [git-blame-ignore-rev] 3a586cf46cb1bb963b93d9546f20273194d15de5 # 2023-11-20 - effigies@gmail.com - STY: ruff --fix smriprep [git-blame-ignore-rev] From f30c2fd749baba31bf16bdd3ce2ab66c98126b16 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 1 Mar 2024 14:10:48 -0500 Subject: [PATCH 07/20] FIX: Ensure templates are pulled on workflow construction --- smriprep/utils/misc.py | 27 +++++++++++++++++++++++++++ smriprep/workflows/anatomical.py | 3 +++ 2 files changed, 30 insertions(+) diff --git a/smriprep/utils/misc.py b/smriprep/utils/misc.py index 3c473b49a2..60e6f726fd 100644 --- a/smriprep/utils/misc.py +++ b/smriprep/utils/misc.py @@ -21,6 +21,11 @@ # https://www.nipreps.org/community/licensing/ # """Self-contained utilities to be used within Function nodes.""" +from pathlib import Path +import typing as ty + +if ty.TYPE_CHECKING: + from niworkflows.utils.spaces import Reference def apply_lut(in_dseg, lut, newpath=None): @@ -95,3 +100,25 @@ def fs_isRunning(subjects_dir, subject_id, mtime_tol=86400, logger=None): if logger: logger.warn(f'Removed "IsRunning*" files found under {subj_dir}') return subjects_dir + + +def get_template_t1w(template: str, sloppy: bool = False) -> Path: + """Query templateflow for the T1w to ensure it is present on the filesystem.""" + import templateflow.api as tf + + spec = {} + _space = template.split(':', 1) + if len(_space) > 1: + spec['cohort'] = _space[1].replace('cohort-', '') + space = _space[0] + + available_res = tf.TF_LAYOUT.get_resolutions(template=space) + if sloppy and 2 in available_res: + res = 2 + elif 1 in available_res: + res = 1 + else: + res = None + spec['resolution'] = res + + return tf.get(space, desc=None, suffix='T1w', **spec) diff --git a/smriprep/workflows/anatomical.py b/smriprep/workflows/anatomical.py index e65620da47..3b1534c960 100644 --- a/smriprep/workflows/anatomical.py +++ b/smriprep/workflows/anatomical.py @@ -973,6 +973,9 @@ def init_anat_fit_wf( templates = [] found_xfms = {} for template in spaces.get_spaces(nonstandard=False, dim=(3,)): + from ..utils.misc import get_template_t1w + + get_template_t1w(template, sloppy) xfms = precomputed.get('transforms', {}).get(template, {}) if set(xfms) != {'forward', 'reverse'}: templates.append(template) From 718843992f8fb1cc558dbfd5c6effa441dc58db4 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 1 Mar 2024 16:31:50 -0500 Subject: [PATCH 08/20] FIX: Ensure sloppy resolution is used --- smriprep/workflows/anatomical.py | 3 ++- smriprep/workflows/outputs.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/smriprep/workflows/anatomical.py b/smriprep/workflows/anatomical.py index 3b1534c960..f8b121b38b 100644 --- a/smriprep/workflows/anatomical.py +++ b/smriprep/workflows/anatomical.py @@ -277,7 +277,7 @@ def init_anat_preproc_wf( omp_nthreads=omp_nthreads, skull_strip_fixed_seed=skull_strip_fixed_seed, ) - template_iterator_wf = init_template_iterator_wf(spaces=spaces) + template_iterator_wf = init_template_iterator_wf(spaces=spaces, sloppy=sloppy) ds_std_volumes_wf = init_ds_anat_volumes_wf( bids_root=bids_root, output_dir=output_dir, @@ -725,6 +725,7 @@ def init_anat_fit_wf( spaces=spaces, freesurfer=freesurfer, output_dir=output_dir, + sloppy=sloppy, ) # fmt:off workflow.connect([ diff --git a/smriprep/workflows/outputs.py b/smriprep/workflows/outputs.py index 82f0f668c8..0ac3fba8df 100644 --- a/smriprep/workflows/outputs.py +++ b/smriprep/workflows/outputs.py @@ -35,10 +35,13 @@ from ..interfaces import DerivativesDataSink from ..interfaces.templateflow import TemplateFlowSelect +if ty.TYPE_CHECKING: + from niworkflows.utils.spaces import SpatialReferences + BIDS_TISSUE_ORDER = ('GM', 'WM', 'CSF') -def init_anat_reports_wf(*, spaces, freesurfer, output_dir, name='anat_reports_wf'): +def init_anat_reports_wf(*, spaces, freesurfer, output_dir, sloppy=False, name='anat_reports_wf'): """ Set up a battery of datasinks to store reports in the right location. @@ -131,7 +134,7 @@ def init_anat_reports_wf(*, spaces, freesurfer, output_dir, name='anat_reports_w # fmt:on if spaces._cached is not None and spaces.cached.references: - template_iterator_wf = init_template_iterator_wf(spaces=spaces) + template_iterator_wf = init_template_iterator_wf(spaces=spaces, sloppy=sloppy) t1w_std = pe.Node( ApplyTransforms( dimension=3, @@ -1112,7 +1115,12 @@ def init_anat_second_derivatives_wf( return workflow -def init_template_iterator_wf(*, spaces, name='template_iterator_wf'): +def init_template_iterator_wf( + *, + spaces: 'SpatialReferences', + sloppy: bool = False, + name='template_iterator_wf' +): """Prepare the necessary components to resample an image to a template space This produces a workflow with an unjoined iterable named "spacesource". @@ -1160,7 +1168,7 @@ def init_template_iterator_wf(*, spaces, name='template_iterator_wf'): run_without_submitting=True, ) select_tpl = pe.Node( - TemplateFlowSelect(resolution=1), name='select_tpl', run_without_submitting=True + TemplateFlowSelect(resolution=2 if sloppy else 1), name='select_tpl', run_without_submitting=True ) # fmt:off From b7573676358de1af34ff758051cd74ebfc7d7cb5 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Wed, 6 Mar 2024 11:49:11 -0500 Subject: [PATCH 09/20] FIX: Determine resolution by sloppy parameter --- smriprep/workflows/outputs.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/smriprep/workflows/outputs.py b/smriprep/workflows/outputs.py index 0ac3fba8df..b99609a71f 100644 --- a/smriprep/workflows/outputs.py +++ b/smriprep/workflows/outputs.py @@ -1168,7 +1168,7 @@ def init_template_iterator_wf( run_without_submitting=True, ) select_tpl = pe.Node( - TemplateFlowSelect(resolution=2 if sloppy else 1), name='select_tpl', run_without_submitting=True + TemplateFlowSelect(), name='select_tpl', run_without_submitting=True ) # fmt:off @@ -1185,7 +1185,7 @@ def init_template_iterator_wf( (spacesource, select_tpl, [ ('space', 'template'), ('cohort', 'cohort'), - (('resolution', _no_native), 'resolution'), + (('resolution', _no_native, sloppy), 'resolution'), ]), (spacesource, outputnode, [ ('space', 'space'), @@ -1251,10 +1251,6 @@ def _pick_cohort(in_template): return [_pick_cohort(v) for v in in_template] -def _fmt(in_template): - return in_template.replace(':', '_') - - def _empty_report(in_file=None): from pathlib import Path @@ -1276,11 +1272,11 @@ def _is_native(value): return value == 'native' -def _no_native(value): +def _no_native(value, sloppy=False): try: return int(value) except (TypeError, ValueError): - return 1 + return 2 if sloppy else 1 def _drop_path(in_path): From 97f4556ecb77cc8bd293c1bfbdbdfe52441effda Mon Sep 17 00:00:00 2001 From: mathiasg Date: Wed, 6 Mar 2024 13:50:53 -0500 Subject: [PATCH 10/20] RF: Strip out template fetching to separate function --- smriprep/interfaces/templateflow.py | 67 +++++++++++++++++------------ smriprep/utils/misc.py | 28 ------------ 2 files changed, 39 insertions(+), 56 deletions(-) diff --git a/smriprep/interfaces/templateflow.py b/smriprep/interfaces/templateflow.py index e7199c8b94..2a95893d48 100644 --- a/smriprep/interfaces/templateflow.py +++ b/smriprep/interfaces/templateflow.py @@ -108,34 +108,9 @@ def _run_interface(self, runtime): if isdefined(self.inputs.cohort): specs['cohort'] = self.inputs.cohort - name = self.inputs.template.strip(':').split(':', 1) - if len(name) > 1: - specs.update( - { - k: v - for modifier in name[1].split(':') - for k, v in [tuple(modifier.split('-'))] - if k not in specs - } - ) - - if specs['resolution'] and not isinstance(specs['resolution'], list): - specs['resolution'] = [specs['resolution']] - - available_resolutions = tf.TF_LAYOUT.get_resolutions(template=name[0]) - if specs['resolution'] and not set(specs['resolution']) & set(available_resolutions): - fallback_res = available_resolutions[0] if available_resolutions else None - LOGGER.warning( - f"Template {name[0]} does not have resolution(s): {specs['resolution']}." - f"Falling back to resolution: {fallback_res}." - ) - specs['resolution'] = fallback_res - - self._results['t1w_file'] = tf.get(name[0], desc=None, suffix='T1w', **specs) - - self._results['brain_mask'] = tf.get( - name[0], desc='brain', suffix='mask', **specs - ) or tf.get(name[0], label='brain', suffix='mask', **specs) + files = fetch_template_files(self.inputs.template, specs) + self._results['t1w_file'] = files['t1w'] + self._results['brain_mask'] = files['mask'] return runtime @@ -186,3 +161,39 @@ def _run_interface(self, runtime): descsplit = desc.split('-') self._results['spec'][descsplit[0]] = descsplit[1] return runtime + + +def fetch_template_files(template: str, specs: dict | None = None) -> dict: + if specs is None: + specs = {} + + name = template.strip(':').split(':', 1) + if len(name) > 1: + specs.update( + { + k: v + for modifier in name[1].split(':') + for k, v in [tuple(modifier.split('-'))] + if k not in specs + } + ) + + if specs['resolution'] and not isinstance(specs['resolution'], list): + specs['resolution'] = [specs['resolution']] + + available_resolutions = tf.TF_LAYOUT.get_resolutions(template=name[0]) + if specs['resolution'] and not set(specs['resolution']) & set(available_resolutions): + fallback_res = available_resolutions[0] if available_resolutions else None + LOGGER.warning( + f"Template {name[0]} does not have resolution(s): {specs['resolution']}." + f"Falling back to resolution: {fallback_res}." + ) + specs['resolution'] = fallback_res + + files = {} + files['t1w'] = tf.get(name[0], desc=None, suffix='T1w', **specs) + files['mask'] = ( + tf.get(name[0], desc='brain', suffix='mask', **specs) + or tf.get(name[0], label='brain', suffix='mask', **specs) + ) + return files diff --git a/smriprep/utils/misc.py b/smriprep/utils/misc.py index 60e6f726fd..2e399a7ad4 100644 --- a/smriprep/utils/misc.py +++ b/smriprep/utils/misc.py @@ -21,12 +21,6 @@ # https://www.nipreps.org/community/licensing/ # """Self-contained utilities to be used within Function nodes.""" -from pathlib import Path -import typing as ty - -if ty.TYPE_CHECKING: - from niworkflows.utils.spaces import Reference - def apply_lut(in_dseg, lut, newpath=None): """Map the input discrete segmentation to a new label set (lookup table, LUT).""" @@ -100,25 +94,3 @@ def fs_isRunning(subjects_dir, subject_id, mtime_tol=86400, logger=None): if logger: logger.warn(f'Removed "IsRunning*" files found under {subj_dir}') return subjects_dir - - -def get_template_t1w(template: str, sloppy: bool = False) -> Path: - """Query templateflow for the T1w to ensure it is present on the filesystem.""" - import templateflow.api as tf - - spec = {} - _space = template.split(':', 1) - if len(_space) > 1: - spec['cohort'] = _space[1].replace('cohort-', '') - space = _space[0] - - available_res = tf.TF_LAYOUT.get_resolutions(template=space) - if sloppy and 2 in available_res: - res = 2 - elif 1 in available_res: - res = 1 - else: - res = None - spec['resolution'] = res - - return tf.get(space, desc=None, suffix='T1w', **spec) From 5166979513efdc3a54a3255bd81abb62a36a1587 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Wed, 6 Mar 2024 14:25:54 -0500 Subject: [PATCH 11/20] FIX: Fetch templates during workflow construction --- smriprep/interfaces/templateflow.py | 17 ++++++++++++++--- smriprep/workflows/anatomical.py | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/smriprep/interfaces/templateflow.py b/smriprep/interfaces/templateflow.py index 2a95893d48..af38fa1b87 100644 --- a/smriprep/interfaces/templateflow.py +++ b/smriprep/interfaces/templateflow.py @@ -163,7 +163,11 @@ def _run_interface(self, runtime): return runtime -def fetch_template_files(template: str, specs: dict | None = None) -> dict: +def fetch_template_files( + template: str, + specs: dict | None = None, + sloppy: bool = False, +) -> dict: if specs is None: specs = {} @@ -178,11 +182,18 @@ def fetch_template_files(template: str, specs: dict | None = None) -> dict: } ) - if specs['resolution'] and not isinstance(specs['resolution'], list): + if res := specs.pop('res', None): + if res != 'native': + specs['resolution'] = res + + if not specs.get('resolution'): + specs['resolution'] = 2 if sloppy else 1 + + if specs.get('resolution') and not isinstance(specs['resolution'], list): specs['resolution'] = [specs['resolution']] available_resolutions = tf.TF_LAYOUT.get_resolutions(template=name[0]) - if specs['resolution'] and not set(specs['resolution']) & set(available_resolutions): + if specs.get('resolution') and not set(specs['resolution']) & set(available_resolutions): fallback_res = available_resolutions[0] if available_resolutions else None LOGGER.warning( f"Template {name[0]} does not have resolution(s): {specs['resolution']}." diff --git a/smriprep/workflows/anatomical.py b/smriprep/workflows/anatomical.py index f8b121b38b..4135923aae 100644 --- a/smriprep/workflows/anatomical.py +++ b/smriprep/workflows/anatomical.py @@ -974,9 +974,9 @@ def init_anat_fit_wf( templates = [] found_xfms = {} for template in spaces.get_spaces(nonstandard=False, dim=(3,)): - from ..utils.misc import get_template_t1w + from smriprep.interfaces.templateflow import fetch_template_files - get_template_t1w(template, sloppy) + fetch_template_files(template, specs=None, sloppy=sloppy) xfms = precomputed.get('transforms', {}).get(template, {}) if set(xfms) != {'forward', 'reverse'}: templates.append(template) From e351f9a4bdeca0744b396e6fef2770e37175cb46 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 7 Mar 2024 11:48:00 -0500 Subject: [PATCH 12/20] RF: Move template fetching to iterator workflow, adhere to style --- smriprep/interfaces/templateflow.py | 5 ++--- smriprep/utils/misc.py | 1 + smriprep/workflows/anatomical.py | 3 --- smriprep/workflows/outputs.py | 14 +++++++------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/smriprep/interfaces/templateflow.py b/smriprep/interfaces/templateflow.py index af38fa1b87..6a466d7d09 100644 --- a/smriprep/interfaces/templateflow.py +++ b/smriprep/interfaces/templateflow.py @@ -203,8 +203,7 @@ def fetch_template_files( files = {} files['t1w'] = tf.get(name[0], desc=None, suffix='T1w', **specs) - files['mask'] = ( - tf.get(name[0], desc='brain', suffix='mask', **specs) - or tf.get(name[0], label='brain', suffix='mask', **specs) + files['mask'] = tf.get(name[0], desc='brain', suffix='mask', **specs) or tf.get( + name[0], label='brain', suffix='mask', **specs ) return files diff --git a/smriprep/utils/misc.py b/smriprep/utils/misc.py index 2e399a7ad4..3c473b49a2 100644 --- a/smriprep/utils/misc.py +++ b/smriprep/utils/misc.py @@ -22,6 +22,7 @@ # """Self-contained utilities to be used within Function nodes.""" + def apply_lut(in_dseg, lut, newpath=None): """Map the input discrete segmentation to a new label set (lookup table, LUT).""" import nibabel as nb diff --git a/smriprep/workflows/anatomical.py b/smriprep/workflows/anatomical.py index 4135923aae..4bdfd285c5 100644 --- a/smriprep/workflows/anatomical.py +++ b/smriprep/workflows/anatomical.py @@ -974,9 +974,6 @@ def init_anat_fit_wf( templates = [] found_xfms = {} for template in spaces.get_spaces(nonstandard=False, dim=(3,)): - from smriprep.interfaces.templateflow import fetch_template_files - - fetch_template_files(template, specs=None, sloppy=sloppy) xfms = precomputed.get('transforms', {}).get(template, {}) if set(xfms) != {'forward', 'reverse'}: templates.append(template) diff --git a/smriprep/workflows/outputs.py b/smriprep/workflows/outputs.py index b99609a71f..9cf2f45ea3 100644 --- a/smriprep/workflows/outputs.py +++ b/smriprep/workflows/outputs.py @@ -1116,10 +1116,7 @@ def init_anat_second_derivatives_wf( def init_template_iterator_wf( - *, - spaces: 'SpatialReferences', - sloppy: bool = False, - name='template_iterator_wf' + *, spaces: 'SpatialReferences', sloppy: bool = False, name='template_iterator_wf' ): """Prepare the necessary components to resample an image to a template space @@ -1130,6 +1127,11 @@ def init_template_iterator_wf( The fields in `outputnode` can be used as if they come from a single template. """ + for template in spaces.get_spaces(nonstandard=False, dim=(3,)): + from smriprep.interfaces.templateflow import fetch_template_files + + fetch_template_files(template, specs=None, sloppy=sloppy) + workflow = pe.Workflow(name=name) inputnode = pe.Node( @@ -1167,9 +1169,7 @@ def init_template_iterator_wf( name='select_xfm', run_without_submitting=True, ) - select_tpl = pe.Node( - TemplateFlowSelect(), name='select_tpl', run_without_submitting=True - ) + select_tpl = pe.Node(TemplateFlowSelect(), name='select_tpl', run_without_submitting=True) # fmt:off workflow.connect([ From 9f13a869266c84afa2f3925eded69a94408625c9 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Thu, 7 Mar 2024 14:45:29 -0500 Subject: [PATCH 13/20] Apply suggestions from code review Co-authored-by: Chris Markiewicz --- smriprep/workflows/outputs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/smriprep/workflows/outputs.py b/smriprep/workflows/outputs.py index 9cf2f45ea3..dee84d17a0 100644 --- a/smriprep/workflows/outputs.py +++ b/smriprep/workflows/outputs.py @@ -33,7 +33,7 @@ from niworkflows.interfaces.utility import KeySelect from ..interfaces import DerivativesDataSink -from ..interfaces.templateflow import TemplateFlowSelect +from ..interfaces.templateflow import TemplateFlowSelect, fetch_template_files if ty.TYPE_CHECKING: from niworkflows.utils.spaces import SpatialReferences @@ -1128,8 +1128,6 @@ def init_template_iterator_wf( The fields in `outputnode` can be used as if they come from a single template. """ for template in spaces.get_spaces(nonstandard=False, dim=(3,)): - from smriprep.interfaces.templateflow import fetch_template_files - fetch_template_files(template, specs=None, sloppy=sloppy) workflow = pe.Workflow(name=name) From 4cf06d3cd0b59e4c1fd9d7d8118d977debc6b5cb Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 8 Mar 2024 13:02:42 -0500 Subject: [PATCH 14/20] DOC: 0.14.0 changelog --- CHANGES.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 6d1fa38462..5a61cf35cd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,18 @@ +0.14.0 (March 08, 2024) +======================= +New feature release in the 0.14.x series. + +This release restores correct handling of cohort identifiers in templates. +A feature release is warranted due to changes in the workflow structure. + +* FIX: Fetch templates during workflow construction (#418) +* FIX: Re-add cohort identifier to template name (#416) +* FIX: Repair FreeSurfer Dependency in Dockerfile (tcsh) (#404) +* FIX: Invert result of skull-strip check in auto mode (#402) +* STY: Adopt ruff for linting and formatting (#397) +* CHORE: Update ruff, ignore certain rules (#419) + + 0.13.2 (December 08, 2023) ========================== Bug fix release in the 0.13.x series. From 74a3745a15f87fe3f565a771e4c8a1625ce992ef Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 6 Mar 2024 14:52:51 -0500 Subject: [PATCH 15/20] Update pyproject requirements and env.yml --- env.yml | 18 +++++++++--------- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/env.yml b/env.yml index 2e99487e37..6c4d96c74c 100644 --- a/env.yml +++ b/env.yml @@ -2,29 +2,29 @@ name: smriprep channels: - https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/public/ - conda-forge -# Update this ~yearly; last updated April 2023 +# Update this ~yearly; last updated March 2024 dependencies: - - python=3.10 + - python=3.11 # Needed for svgo and bids-validator; consider moving to deno - - nodejs=18 + - nodejs=20 # Intel Math Kernel Library for numpy - - mkl=2022.1 + - mkl=2023.2 - mkl-service=2.4 # Base scientific python stack; required by FSL, so pinned here - numpy=1.26 - scipy=1.11 - matplotlib=3.8 - - pandas=2.1 + - pandas=2.2 - h5py=3.10 # Dependencies compiled against numpy, best to stick with conda - scikit-image=0.22 - - scikit-learn=1.3 + - scikit-learn=1.4 # Utilities - - graphviz=6.0 + - graphviz=9.0 - pandoc=3.1 # Workflow dependencies: ANTs - - ants=2.5.0 - # Workflow dependencies: FSL (versions pinned in 6.0.6.2) + - ants=2.5 + # Workflow dependencies: FSL (versions pinned in 6.0.7.7) - fsl-bet2=2111.4 - fsl-flirt=2111.2 - fsl-fast4=2111.3 diff --git a/pyproject.toml b/pyproject.toml index 0bd878e582..b7b3e3a16b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "matplotlib >= 2.2.0", "nibabel >= 4.0.1", "nipype >= 1.7.0", - "niworkflows @ git+https://github.com/nipreps/niworkflows.git@master", + "niworkflows >= 1.10.1", "numpy", "packaging", "pybids >= 0.11.1", From 14ca542d240387842ac5e59599f86ea4a3e1340b Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 6 Mar 2024 14:53:12 -0500 Subject: [PATCH 16/20] chore(pin): echo "scipy<1.12" | uv pip compile --upgrade pyproject.toml -c - -p 3.11 -o requirements.txt --- requirements.txt | 93 +++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/requirements.txt b/requirements.txt index fe4fbe0067..2d90f82d01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,12 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --strip-extras -# +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -c - -p 3.11 -o requirements.txt astor==0.8.1 # via formulaic -attrs==23.1.0 +attrs==23.2.0 # via niworkflows -bids-validator==1.13.1 +bids-validator==1.14.1 # via pybids -certifi==2023.11.17 +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests @@ -30,25 +26,24 @@ etelemetry==0.3.1 # via nipype filelock==3.13.1 # via nipype -fonttools==4.44.3 +fonttools==4.49.0 # via matplotlib formulaic==0.5.2 # via pybids -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy h5py==3.10.0 # via nitransforms -idna==3.4 +idna==3.6 # via requests -imageio==2.32.0 +imageio==2.34.0 # via scikit-image indexed-gzip==1.8.7 - # via smriprep (pyproject.toml) interface-meta==1.3.0 # via formulaic isodate==0.6.1 # via rdflib -jinja2==3.1.2 +jinja2==3.1.3 # via niworkflows joblib==1.3.2 # via @@ -59,50 +54,43 @@ kiwisolver==1.4.5 lazy-loader==0.3 # via scikit-image lockfile==0.12.2 - # via smriprep (pyproject.toml) looseversion==1.3.0 # via # nipype # niworkflows - # smriprep (pyproject.toml) -lxml==4.9.3 +lxml==5.1.0 # via # nilearn # prov # svgutils -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -matplotlib==3.8.2 +matplotlib==3.8.3 # via # niworkflows # seaborn - # smriprep (pyproject.toml) networkx==3.2.1 # via # nipype # prov # scikit-image -nibabel==5.1.0 +nibabel==5.2.1 # via # nilearn # nipype # nitransforms # niworkflows # pybids - # smriprep (pyproject.toml) -nilearn==0.10.2 +nilearn==0.10.3 # via niworkflows nipype==1.8.6 - # via - # niworkflows - # smriprep (pyproject.toml) + # via niworkflows nitransforms==23.0.1 # via niworkflows -niworkflows==1.9.0 - # via smriprep (pyproject.toml) +niworkflows==1.10.1 num2words==0.5.13 # via pybids -numpy==1.26.2 +numpy==1.26.4 # via # contourpy # formulaic @@ -120,7 +108,6 @@ numpy==1.26.2 # scikit-learn # scipy # seaborn - # smriprep (pyproject.toml) # tifffile packaging==23.2 # via @@ -131,45 +118,41 @@ packaging==23.2 # nipype # niworkflows # scikit-image - # smriprep (pyproject.toml) -pandas==2.1.3 +pandas==2.2.1 # via # formulaic # nilearn # niworkflows # pybids # seaborn -pillow==10.0.1 +pillow==10.2.0 # via # imageio # matplotlib # scikit-image prov==2.0.0 # via nipype -pybids==0.16.3 +pybids==0.16.4 # via # niworkflows - # smriprep (pyproject.toml) # templateflow -pydot==1.4.2 +pydot==2.0.0 # via nipype -pyparsing==3.1.1 +pyparsing==3.1.2 # via # matplotlib # pydot # rdflib -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # matplotlib # nipype # pandas # prov -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 - # via - # niworkflows - # smriprep (pyproject.toml) + # via niworkflows rdflib==7.0.0 # via # nipype @@ -181,7 +164,7 @@ requests==2.31.0 # templateflow scikit-image==0.22.0 # via niworkflows -scikit-learn==1.3.2 +scikit-learn==1.4.1.post1 # via nilearn scipy==1.11.4 # via @@ -193,7 +176,7 @@ scipy==1.11.4 # pybids # scikit-image # scikit-learn -seaborn==0.13.0 +seaborn==0.13.2 # via niworkflows simplejson==3.19.2 # via nipype @@ -201,19 +184,17 @@ six==1.16.0 # via # isodate # python-dateutil -sqlalchemy==2.0.23 +sqlalchemy==2.0.28 # via pybids svgutils==0.3.4 # via niworkflows -templateflow==23.1.0 - # via - # niworkflows - # smriprep (pyproject.toml) -threadpoolctl==3.2.0 +templateflow==24.0.0 + # via niworkflows +threadpoolctl==3.3.0 # via scikit-learn -tifffile==2023.9.26 +tifffile==2024.2.12 # via scikit-image -tqdm==4.66.1 +tqdm==4.66.2 # via templateflow traits==6.3.2 # via @@ -221,13 +202,13 @@ traits==6.3.2 # niworkflows transforms3d==0.4.1 # via niworkflows -typing-extensions==4.8.0 +typing-extensions==4.10.0 # via # formulaic # sqlalchemy -tzdata==2023.3 +tzdata==2024.1 # via pandas -urllib3==2.1.0 +urllib3==2.2.1 # via requests wrapt==1.16.0 # via formulaic From 201d28838e1975af7ecf3165402b028667461886 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 8 Mar 2024 13:47:41 -0500 Subject: [PATCH 17/20] chore(build): Sync Dockerfile to fmriprep --- Dockerfile | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 359528bec6..986380a701 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # # MIT License # -# Copyright (c) 2023 The NiPreps Developers +# Copyright (c) The NiPreps Developers # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,17 +23,17 @@ # SOFTWARE. # Ubuntu 22.04 LTS - Jammy -ARG BASE_IMAGE=ubuntu:jammy-20230308 +ARG BASE_IMAGE=ubuntu:jammy-20240125 # -# sMRIPrep wheel +# Build wheel # FROM python:slim AS src RUN pip install build RUN apt-get update && \ apt-get install -y --no-install-recommends git -COPY . /src/smriprep -RUN python -m build /src/smriprep +COPY . /src +RUN python -m build /src # # Download stages @@ -41,6 +41,8 @@ RUN python -m build /src/smriprep # Utilities for downloading packages FROM ${BASE_IMAGE} as downloader +# Bump the date to current to refresh curl/certificates/etc +RUN echo "2023.07.20" RUN apt-get update && \ apt-get install -y --no-install-recommends \ binutils \ @@ -67,19 +69,30 @@ RUN mkdir /opt/workbench && \ # Micromamba FROM downloader as micromamba + +# Install a C compiler to build extensions when needed. +# traits<6.4 wheels are not available for Python 3.11+, but build easily. +RUN apt-get update && \ + apt-get install -y --no-install-recommends build-essential && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + WORKDIR / # Bump the date to current to force update micromamba -RUN echo "2023.04.05" +RUN echo "2024.03.08" RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba ENV MAMBA_ROOT_PREFIX="/opt/conda" COPY env.yml /tmp/env.yml COPY requirements.txt /tmp/requirements.txt +WORKDIR /tmp RUN micromamba create -y -f /tmp/env.yml && \ micromamba clean -y -a -ENV PATH="/opt/conda/envs/smriprep/bin:$PATH" -RUN /opt/conda/envs/smriprep/bin/npm install -g svgo@^3.0 bids-validator@^1.13 && \ +# UV_USE_IO_URING for apparent race-condition (https://github.com/nodejs/node/issues/48444) +# Check if this is still necessary when updating the base image. +ENV PATH="/opt/conda/envs/smriprep/bin:$PATH" \ + UV_USE_IO_URING=0 +RUN npm install -g svgo@^3.2.0 bids-validator@^1.14.0 && \ rm -r ~/.npm # @@ -179,7 +192,7 @@ ENV MKL_NUM_THREADS=1 \ OMP_NUM_THREADS=1 # Installing SMRIPREP -COPY --from=src /src/smriprep/dist/*.whl . +COPY --from=src /src/dist/*.whl . RUN pip install --no-cache-dir $( ls *.whl )[telemetry,test] RUN find $HOME -type d -exec chmod go=u {} + && \ From 688df7d015588ec3fea2e6774e97bc34ec4a9e8c Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 8 Mar 2024 15:32:42 -0500 Subject: [PATCH 18/20] CI: Bump cimg, simplify pyenv, reset cache --- .circleci/config.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ace45f1b1a..3854847095 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,14 +4,14 @@ _machine_defaults: &machine_defaults TZ: "/usr/share/zoneinfo/America/Los_Angeles" SCRATCH: "/scratch" machine: - image: ubuntu-2204:2023.04.2 + image: default docker_layer_caching: true working_directory: /tmp/src/smriprep resource_class: large _python_defaults: &python_defaults docker: - - image: cimg/python:3.10.9 + - image: cimg/python:3.12.2 auth: username: $DOCKER_USER password: $DOCKER_PAT @@ -115,8 +115,7 @@ jobs: name: Build Docker image no_output_timeout: 60m command: | - export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) - pyenv local $PY3 + pyenv local 3 pip install hatch # Get version, update files. THISVERSION=$( hatch version ) @@ -139,8 +138,7 @@ jobs: - run: name: Check Docker image command: | - export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) - pyenv local $PY3 + pyenv local 3 # Get version, update files. THISVERSION=$( hatch version ) BUILT_VERSION=$( docker run --rm nipreps/smriprep:latest --version ) @@ -309,8 +307,7 @@ jobs: - run: name: Test smriprep-wrapper (Python 2) command: | - export PY2=$( pyenv versions | awk '/^\* 2/ { print $2 }' ) - pyenv local $PY2 + pyenv local 2.7 echo -n "Python version: " python --version pip install --upgrade "pip<21" @@ -322,8 +319,7 @@ jobs: - run: name: Test smriprep-wrapper (Python 3) command: | - export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) - pyenv local $PY3 + pyenv local 3 echo -n "Python version: " python --version pip install --upgrade pip setuptools @@ -440,10 +436,10 @@ jobs: - run: *pull_from_registry - restore_cache: keys: - - ds005-anat-v0-{{ .Branch }}-{{ epoch }} - - ds005-anat-v0-{{ .Branch }} - - ds005-anat-v0-master - - ds005-anat-v0-next + - ds005-anat-v1-{{ .Branch }}-{{ epoch }} + - ds005-anat-v1-{{ .Branch }} + - ds005-anat-v1-master + - ds005-anat-v1-next - restore_cache: keys: - testdata-v2-{{ .Branch }}-{{ epoch }} @@ -500,7 +496,7 @@ jobs: rm -rf /tmp/ds005/work/reportlets rm -rf /tmp/ds005/work/smriprep_wf/fsdir_run_*/ - save_cache: - key: ds005-anat-v0-{{ .Branch }}-{{ epoch }} + key: ds005-anat-v1-{{ .Branch }}-{{ epoch }} paths: - /tmp/ds005/work From 7d777a30b69354af66d063ac519d5b4b1d42931c Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 8 Mar 2024 16:07:29 -0500 Subject: [PATCH 19/20] CI: Sort expected outputs for consistency --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3854847095..3c67a2d7ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -505,7 +505,8 @@ jobs: command: | mkdir -p /tmp/ds005/test find /tmp/ds005/derivatives -name "*" ! -path "*/figures*" -print | sed s+/tmp/ds005/derivatives/++ | sort > /tmp/ds005/test/outputs.out - diff /tmp/src/smriprep/.circleci/ds005_outputs.txt /tmp/ds005/test/outputs.out + sort /tmp/src/smriprep/.circleci/ds005_outputs.txt > /tmp/ds005/test/expected.out + diff /tmp/ds005/test/{expected,outputs}.out exit $? - store_artifacts: path: /tmp/ds005/derivatives @@ -636,7 +637,8 @@ jobs: command: | mkdir -p /tmp/ds054/test find /tmp/ds054/derivatives -path */figures -prune -o -name "*" -print | sed s+/tmp/ds054/derivatives/++ | sort > /tmp/ds054/test/outputs.out - diff /tmp/src/smriprep/.circleci/ds054_outputs.txt /tmp/ds054/test/outputs.out + sort /tmp/src/smriprep/.circleci/ds054_outputs.txt > /tmp/ds054/test/expected.out + diff /tmp/ds054/test/{expected,outputs}.out exit $? - run: name: Clean working directory From 45c2e2fc088839979f18b83be630a6f9d515b416 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Mon, 11 Mar 2024 09:37:56 -0400 Subject: [PATCH 20/20] rel(0.14.0): update changes [skip ci] --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5a61cf35cd..25155a24d1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -0.14.0 (March 08, 2024) +0.14.0 (March 11, 2024) ======================= New feature release in the 0.14.x series.