diff --git a/bidscoin/__init__.py b/bidscoin/__init__.py
index 6f5596bd..0774cb90 100644
--- a/bidscoin/__init__.py
+++ b/bidscoin/__init__.py
@@ -27,7 +27,7 @@
import subprocess
from pathlib import Path
from importlib import metadata
-from typing import Tuple, Union, List
+from typing import Union
from logging import getLogger
from .due import due, Doi
try:
@@ -101,7 +101,7 @@
path='bidscoin', version=__version__, cite_module=True, tags=['reference-implementation'])
-def check_version() -> Tuple[str, Union[bool, None], str]:
+def check_version() -> tuple[str, Union[bool, None], str]:
"""
Compares the BIDSCOIN version from the local metadata to the remote pypi repository
@@ -141,7 +141,7 @@ def is_hidden(path: Path):
return hidden
-def lsdirs(folder: Path, wildcard: str='*') -> List[Path]:
+def lsdirs(folder: Path, wildcard: str='*') -> list[Path]:
"""
Gets all sorted directories in a folder, ignores files. Foldernames starting with a dot are considered hidden and will be skipped
diff --git a/bidscoin/bcoin.py b/bidscoin/bcoin.py
index 538e7b16..9631b16a 100755
--- a/bidscoin/bcoin.py
+++ b/bidscoin/bcoin.py
@@ -19,7 +19,7 @@
from importlib.metadata import entry_points
from importlib.util import spec_from_file_location, module_from_spec
from pathlib import Path
-from typing import Tuple, Union, List
+from typing import Union
from ruamel.yaml import YAML
from tqdm import tqdm
from tqdm.contrib.logging import logging_redirect_tqdm
@@ -241,7 +241,7 @@ def list_executables(show: bool=False) -> list:
return scripts
-def list_plugins(show: bool=False) -> Tuple[List[Path], List[Path]]:
+def list_plugins(show: bool=False) -> tuple[list[Path], list[Path]]:
"""
:param show: Print the template bidsmaps and installed plugins if True
:return: List of the installed plugins and template bidsmaps
@@ -264,7 +264,7 @@ def list_plugins(show: bool=False) -> Tuple[List[Path], List[Path]]:
return plugins, templates
-def install_plugins(filenames: List[str]=()) -> None:
+def install_plugins(filenames: list[str]=()) -> None:
"""
Installs template bidsmaps and plugins and adds the plugin Options and data format section to the default template bidsmap
@@ -296,7 +296,7 @@ def install_plugins(filenames: List[str]=()) -> None:
continue
# Check if we can import the plugin
- module = import_plugin(file, ('bidsmapper_plugin', 'bidscoiner_plugin'))
+ module = import_plugin(file)
if not module:
LOGGER.error(f"Plugin failure, please re-install a valid version of '{file.name}'")
continue
@@ -316,7 +316,7 @@ def install_plugins(filenames: List[str]=()) -> None:
LOGGER.success(f"The '{file.name}' plugin was successfully installed")
-def uninstall_plugins(filenames: List[str]=(), wipe: bool=True) -> None:
+def uninstall_plugins(filenames: list[str]=(), wipe: bool=True) -> None:
"""
Uninstalls template bidsmaps and plugins and removes the plugin Options and data format section from the default template bidsmap
@@ -338,7 +338,7 @@ def uninstall_plugins(filenames: List[str]=(), wipe: bool=True) -> None:
# First check if we can import the plugin
if file.suffix == '.py':
- module = import_plugin(pluginfolder/file.name, ('bidsmapper_plugin', 'bidscoiner_plugin'))
+ module = import_plugin(pluginfolder/file.name)
else:
module = None
@@ -373,13 +373,13 @@ def uninstall_plugins(filenames: List[str]=(), wipe: bool=True) -> None:
@lru_cache()
-def import_plugin(plugin: Union[Path,str], functions: tuple=()) -> Union[types.ModuleType, None]:
+def import_plugin(plugin: Union[Path,str], classes: tuple=('Interface',)) -> Union[types.ModuleType, None]:
"""
Imports the plugin if it contains any of the specified functions
- :param plugin: Name of the plugin in the bidscoin "plugins" folder or the fullpath name
- :param functions: List of functions of which at least one of them should be present in the plugin
- :return: The imported plugin-module
+ :param plugin: Name of the plugin in the bidscoin "plugins" folder or the fullpath name
+ :param classes: List of classes of which at least one of them should be present in the plugin
+ :return: The imported plugin-module
"""
if not plugin: return
@@ -401,17 +401,17 @@ def import_plugin(plugin: Union[Path,str], functions: tuple=()) -> Union[types.M
module = module_from_spec(spec)
spec.loader.exec_module(module)
- functionsfound = []
- for function in functions:
- if not hasattr(module, function):
- LOGGER.verbose(f"Could not find '{function}' in the '{plugin}' plugin")
- elif not callable(getattr(module, function)):
- LOGGER.error(f"'The {function}' attribute in the '{plugin}' plugin is not callable")
+ classesfound = []
+ for klass in classes:
+ if not hasattr(module, klass):
+ LOGGER.verbose(f"Could not find '{klass}' in the '{plugin}' plugin")
+ elif not callable(getattr(module, klass)):
+ LOGGER.error(f"'The {klass}' attribute in the '{plugin}' plugin is not callable")
else:
- functionsfound.append(function)
+ classesfound.append(klass)
- if functions and not functionsfound:
- LOGGER.bcdebug(f"Plugin '{plugin}' does not contain {functions} functions")
+ if classes and not classesfound:
+ LOGGER.bcdebug(f"Plugin '{plugin}' does not contain {classes} classes")
else:
return module
@@ -432,26 +432,22 @@ def test_plugin(plugin: Union[Path,str], options: dict) -> int:
LOGGER.info(f"--------- Testing the '{plugin}' plugin ---------")
- # First test to see if we can import the core plugin methods
- module = import_plugin(plugin, ('bidsmapper_plugin','bidscoiner_plugin'))
+ # First test to see if we can import the plugin interface
+ module = import_plugin(plugin)
if module is None:
return 1
# Then run the plugin's own 'test' routine (if implemented)
- if hasattr(module, 'test') and callable(getattr(module, 'test')):
- try:
- returncode = module.test(options)
- if returncode == 0:
- LOGGER.success(f"The '{plugin}' plugin functioned correctly")
- else:
- LOGGER.warning(f"The '{plugin}' plugin did not function correctly")
- return returncode
- except Exception as pluginerror:
- LOGGER.error(f"Could not run {plugin}.test(options):\n{pluginerror}")
- return 1
- else:
- LOGGER.info(f"The '{plugin}' did not have a test routine")
- return 0
+ try:
+ returncode = module.Interface().test(options)
+ if returncode == 0:
+ LOGGER.success(f"The '{plugin}' plugin functioned correctly")
+ else:
+ LOGGER.warning(f"The '{plugin}' plugin did not function correctly")
+ return returncode
+ except Exception as pluginerror:
+ LOGGER.error(f"Could not run {plugin}.test(options):\n{pluginerror}")
+ return 1
def test_bidsmap(bidsmapfile: str):
diff --git a/bidscoin/bids.py b/bidscoin/bids.py
index 710c4b09..b146ab8e 100644
--- a/bidscoin/bids.py
+++ b/bidscoin/bids.py
@@ -21,7 +21,7 @@
from fnmatch import fnmatch
from functools import lru_cache
from pathlib import Path
-from typing import List, Set, Tuple, Union, Dict, Any, Iterable, NewType
+from typing import Union, Any, Iterable, NewType
from pydicom import dcmread, fileset, config
from importlib.util import find_spec
if find_spec('bidscoin') is None:
@@ -32,17 +32,17 @@
from bidscoin.plugins import EventsParser
from ruamel.yaml import YAML
yaml = YAML()
-yaml.representer.ignore_aliases = lambda *data: True # Expand aliases (https://stackoverflow.com/questions/58091449/disabling-alias-for-yaml-file-in-python)
+yaml.representer.ignore_aliases = lambda *data: True # Expand aliases (https://stackoverflow.com/questions/58091449/disabling-alias-for-yaml-file-in-python)
config.INVALID_KEY_BEHAVIOR = 'IGNORE'
# Define custom data types (replace with proper classes or TypeAlias of Python >= 3.10)
-Plugin = NewType('Plugin', Dict[str, Any])
-Plugins = NewType('Plugin', Dict[str, Plugin])
-Options = NewType('Options', Dict[str, Any])
-Properties = NewType('Properties', Dict[str, Any])
-Attributes = NewType('Attributes', Dict[str, Any])
-Bids = NewType('Bids', Dict[str, Any])
-Meta = NewType('Meta', Dict[str, Any])
+Plugin = NewType('Plugin', dict[str, Any])
+Plugins = NewType('Plugin', dict[str, Plugin])
+Options = NewType('Options', dict[str, Any])
+Properties = NewType('Properties', dict[str, Any])
+Attributes = NewType('Attributes', dict[str, Any])
+Bids = NewType('Bids', dict[str, Any])
+Meta = NewType('Meta', dict[str, Any])
LOGGER = logging.getLogger(__name__)
@@ -125,10 +125,10 @@ def has_support(self) -> str:
return ''
for plugin, options in self.plugins.items():
- module = bcoin.import_plugin(plugin, ('has_support',))
+ module = bcoin.import_plugin(plugin)
if module:
try:
- supported = module.has_support(self.path, self.dataformat)
+ supported = module.Interface().has_support(self.path, self.dataformat)
except Exception as moderror:
supported = ''
LOGGER.exception(f"The {plugin} plugin crashed while reading {self.path}\n{moderror}")
@@ -226,9 +226,9 @@ def attributes(self, attributekey: str, validregexp: bool=False, cache: bool=Tru
elif self.dataformat or self.has_support():
for plugin, options in self.plugins.items():
- module = bcoin.import_plugin(plugin, ('get_attribute',))
+ module = bcoin.import_plugin(plugin)
if module:
- attributeval = module.get_attribute(self.dataformat, self.path, attributekey, options)
+ attributeval = module.Interface().get_attribute(self.dataformat, self.path, attributekey, options)
attributeval = str(attributeval) if attributeval is not None else ''
if attributeval:
break
@@ -278,7 +278,7 @@ def _extattributes(self) -> Attributes:
return Attributes(attributes)
- def subid_sesid(self, subid: str=None, sesid: str=None) -> Tuple[str, str]:
+ def subid_sesid(self, subid: str=None, sesid: str=None) -> tuple[str, str]:
"""
Extract the cleaned-up subid and sesid from the datasource properties or attributes
@@ -447,7 +447,7 @@ def __eq__(self, other):
else:
return NotImplemented
- def check(self, checks: Tuple[bool, bool, bool]=(False, False, False)) -> Tuple[Union[bool, None], Union[bool, None], Union[bool, None]]:
+ def check(self, checks: tuple[bool, bool, bool]=(False, False, False)) -> tuple[Union[bool, None], Union[bool, None], Union[bool, None]]:
"""
Check run for required and optional entities using the BIDS schema files
@@ -607,7 +607,7 @@ def bidsname(self, subid: str='unknown', sesid: str='', validkeys: bool=False, r
return bidsname
- def increment_runindex(self, outfolder: Path, bidsname: str, scans_table: pd.DataFrame=None, targets: Set[Path]=()) -> str:
+ def increment_runindex(self, outfolder: Path, bidsname: str, scans_table: pd.DataFrame=None, targets: set[Path]=()) -> str:
"""
Checks if a file with the same bidsname already exists in the folder and then increments the dynamic runindex
(if any) until no such file is found.
@@ -721,7 +721,7 @@ def __hash__(self):
return hash(str(self))
@property
- def runitems(self) -> List[RunItem]:
+ def runitems(self) -> list[RunItem]:
"""Returns a list of the RunItem objects for this datatype"""
return [RunItem(self.dataformat, self.datatype, rundata, self.options, self.plugins) for rundata in self._data]
@@ -836,7 +836,7 @@ def session(self, value: str):
self._data['session'] = value
@property
- def datatypes(self) -> List[DataType]:
+ def datatypes(self) -> list[DataType]:
"""Gets a list of DataType objects for the dataformat"""
return [DataType(self.dataformat, datatype, self._data[datatype], self.options, self.plugins) for datatype in self._data if datatype not in ('subject', 'session')]
@@ -881,7 +881,7 @@ def delete_runs(self, datatype: Union[str, DataType]=''):
class BidsMap:
"""Reads and writes mapping heuristics from the bidsmap YAML-file"""
- def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterable[Union[Path,str]]=(), checks: Tuple[bool,bool,bool]=(True,True,True)):
+ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterable[Union[Path,str]]=(), checks: tuple[bool,bool,bool]=(True,True,True)):
"""
Read and standardize the bidsmap (i.e. add missing information and perform checks). If yamlfile is not fullpath, then 'folder' is first searched before
the default 'heuristics'. If yamfile is empty, then first 'bidsmap.yaml' is searched for, then 'bidsmap_template'. So fullpath
@@ -1119,7 +1119,7 @@ def validate(self, level: int=1) -> bool:
return valid
- def check(self, checks: Tuple[bool, bool, bool]=(True, True, True)) -> Tuple[Union[bool, None], Union[bool, None], Union[bool, None]]:
+ def check(self, checks: tuple[bool, bool, bool]=(True, True, True)) -> tuple[Union[bool, None], Union[bool, None], Union[bool, None]]:
"""
Check all non-ignored runs in the bidsmap for required and optional entities using the BIDS schema files
@@ -1207,7 +1207,7 @@ def check_template(self) -> bool:
return valid
- def dir(self, dataformat: Union[str, DataFormat]) -> List[Path]:
+ def dir(self, dataformat: Union[str, DataFormat]) -> list[Path]:
"""
Make a provenance list of all the runs in the bidsmap[dataformat]
@@ -1268,7 +1268,7 @@ def exist_run(self, runitem: RunItem, datatype: Union[str, DataType]='') -> bool
return False
- def get_matching_run(self, sourcefile: Union[str, Path], dataformat, runtime=False) -> Tuple[RunItem, str]:
+ def get_matching_run(self, sourcefile: Union[str, Path], dataformat, runtime=False) -> tuple[RunItem, str]:
"""
Find the first run in the bidsmap with properties and attributes that match with the data source. Only non-empty
properties and attributes are matched, except when runtime is True, then the empty attributes are also matched.
@@ -1534,7 +1534,7 @@ def update(self, source_datatype: Union[str, DataType], runitem: RunItem):
LOGGER.error(f"Number of runs in bidsmap['{runitem.dataformat}'] changed unexpectedly: {num_runs_in} -> {num_runs_out}")
-def unpack(sesfolder: Path, wildcard: str='', workfolder: Path='', _subprefix: Union[str,None]='') -> Tuple[Set[Path], bool]:
+def unpack(sesfolder: Path, wildcard: str='', workfolder: Path='', _subprefix: Union[str,None]='') -> tuple[set[Path], bool]:
"""
Unpacks and sorts DICOM files in sourcefolder to a temporary folder if sourcefolder contains a DICOMDIR file or .tar.gz, .gz or .zip files
@@ -1572,7 +1572,7 @@ def unpack(sesfolder: Path, wildcard: str='', workfolder: Path='', _subprefix: U
shutil.copytree(sesfolder, worksesfolder, dirs_exist_ok=True)
# Unpack the zip/tarball files in the temporary folder
- sessions: Set[Path] = set()
+ sessions: set[Path] = set()
for tarzipfile in [worksesfolder/tarzipfile.name for tarzipfile in tarzipfiles]:
LOGGER.info(f"Unpacking: {tarzipfile.name} -> {worksesfolder}")
try:
@@ -1674,7 +1674,7 @@ def get_dicomfile(folder: Path, index: int=0) -> Path:
return Path()
-def get_parfiles(folder: Path) -> List[Path]:
+def get_parfiles(folder: Path) -> list[Path]:
"""
Gets the Philips PAR-file from the folder
@@ -1685,7 +1685,7 @@ def get_parfiles(folder: Path) -> List[Path]:
if is_hidden(Path(folder.name)):
return []
- parfiles: List[Path] = []
+ parfiles: list[Path] = []
for file in sorted(folder.iterdir()):
if not is_hidden(file.relative_to(folder)) and is_parfile(file):
parfiles.append(file)
@@ -2334,7 +2334,7 @@ def check_runindices(session: Path) -> bool:
return True
-def limitmatches(fmap: str, matches: List[str], limits: str, niifiles: Set[str], scans_table: pd.DataFrame):
+def limitmatches(fmap: str, matches: list[str], limits: str, niifiles: set[str], scans_table: pd.DataFrame):
"""
Helper function for addmetadata() to check if there are multiple fieldmap runs and get the lower- and upperbound from
the AcquisitionTime to bound the grand list of matches to adjacent runs. The resulting list is appended to niifiles
@@ -2518,7 +2518,7 @@ def addmetadata(bidsses: Path):
json.dump(jsondata, sidecar, indent=4)
-def poolmetadata(datasource: DataSource, targetmeta: Path, usermeta: Meta, extensions: Iterable, sourcemeta: Path=Path()) -> Meta:
+def poolmetadata(datasource: DataSource, targetmeta: Path, usermeta: Meta, metaext: Iterable, sourcemeta: Path=Path()) -> Meta:
"""
Load the metadata from the target (json sidecar), then add metadata from the source (json sidecar) and finally add
the user metadata (meta table). Source metadata other than json sidecars are copied over to the target folder. Special
@@ -2529,7 +2529,7 @@ def poolmetadata(datasource: DataSource, targetmeta: Path, usermeta: Meta, exten
:param datasource: The data source from which dynamic values are read
:param targetmeta: The filepath of the target data file with meta-data
:param usermeta: A user metadata dict, e.g. the meta table from a run-item
- :param extensions: A list of file extensions of the source metadata files, e.g. as specified in bidsmap.plugins['plugin']['meta']
+ :param metaext: A list of file extensions of the source metadata files, e.g. as specified in bidsmap.plugins['plugin']['meta']
:param sourcemeta: The filepath of the source data file with associated/equally named meta-data files (name may include wildcards). Leave empty to use datasource.path
:return: The combined target + source + user metadata
"""
@@ -2544,7 +2544,7 @@ def poolmetadata(datasource: DataSource, targetmeta: Path, usermeta: Meta, exten
metapool = json.load(json_fid)
# Add the source metadata to the metadict or copy it over
- for ext in extensions:
+ for ext in metaext:
for sourcefile in sourcemeta.parent.glob(sourcemeta.with_suffix('').with_suffix(ext).name):
LOGGER.verbose(f"Copying source data from: '{sourcefile}''")
@@ -2600,7 +2600,7 @@ def poolmetadata(datasource: DataSource, targetmeta: Path, usermeta: Meta, exten
return Meta(metapool)
-def addparticipant(participants_tsv: Path, subid: str='', sesid: str='', data: dict=None, dryrun: bool=False) -> Tuple[pd.DataFrame, dict]:
+def addparticipant(participants_tsv: Path, subid: str='', sesid: str='', data: dict=None, dryrun: bool=False) -> tuple[pd.DataFrame, dict]:
"""
Read/create and/or add (if it's not there yet) a participant to the participants.tsv/.json file
diff --git a/bidscoin/bidscoiner.py b/bidscoin/bidscoiner.py
index e569c2ec..08b98c74 100755
--- a/bidscoin/bidscoiner.py
+++ b/bidscoin/bidscoiner.py
@@ -98,7 +98,7 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
return
# Load the data conversion plugins
- plugins = [plugin for name in bidsmap.plugins if (plugin := bcoin.import_plugin(name, ('bidscoiner_plugin',)))]
+ plugins = [plugin for name in bidsmap.plugins if (plugin := bcoin.import_plugin(name))]
if not plugins:
LOGGER.warning(f"The plugins listed in your bidsmap['Options'] did not have a usable `bidscoiner_plugin` function, nothing to do")
LOGGER.info('-------------- FINISHED! ------------')
@@ -274,7 +274,7 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
for plugin in plugins:
LOGGER.verbose(f"Executing plugin: {Path(plugin.__file__).stem}")
trackusage(Path(plugin.__file__).stem)
- personals = plugin.bidscoiner_plugin(sesfolder, bidsmap, bidssession)
+ personals = plugin.Interface().bidscoiner(sesfolder, bidsmap, bidssession)
# Add a subject row to the participants table (if there is any data)
if next(bidssession.rglob('*.json'), None):
diff --git a/bidscoin/bidseditor.py b/bidscoin/bidseditor.py
index 7506b8a9..af833e9f 100755
--- a/bidscoin/bidseditor.py
+++ b/bidscoin/bidseditor.py
@@ -10,7 +10,7 @@
import csv
import nibabel as nib
from bids_validator import BIDSValidator
-from typing import Union, List, Dict
+from typing import Union
from pydicom import dcmread, datadict, config
from pathlib import Path
from functools import partial
@@ -163,9 +163,9 @@ def __init__(self, bidsfolder: Path, input_bidsmap: BidsMap, template_bidsmap: B
self.datasaved = datasaved
"""True if data has been saved on disk"""
self.dataformats = [dataformat.dataformat for dataformat in input_bidsmap.dataformats if input_bidsmap.dir(dataformat)]
- self.bidsignore: List[str] = input_bidsmap.options['bidsignore']
- self.unknowndatatypes: List[str] = input_bidsmap.options['unknowntypes']
- self.ignoredatatypes: List[str] = input_bidsmap.options['ignoretypes']
+ self.bidsignore: list[str] = input_bidsmap.options['bidsignore']
+ self.unknowndatatypes: list[str] = input_bidsmap.options['unknowntypes']
+ self.ignoredatatypes: list[str] = input_bidsmap.options['ignoretypes']
# Set up the tabs, add the tables and put the bidsmap data in them
tabwidget = self.tabwidget = QtWidgets.QTabWidget()
@@ -253,9 +253,9 @@ def samples_menu(self, pos):
rowindexes = [index.row() for index in table.selectedIndexes() if index.column() == colindex]
if rowindexes and colindex in (-1, 0, 4): # User clicked the index, the edit-button or elsewhere (i.e. not on an activated widget)
return
- runitems: List[RunItem] = []
- subids: List[str] = []
- sesids: List[str] = []
+ runitems: list[RunItem] = []
+ subids: list[str] = []
+ sesids: list[str] = []
for index in rowindexes:
datatype = table.item(index, 2).text()
provenance = table.item(index, 5).text()
@@ -1000,8 +1000,8 @@ def __init__(self, runitem: RunItem, bidsmap: BidsMap, template_bidsmap: BidsMap
self.datasource = runitem.datasource
self.dataformat = runitem.dataformat
"""The data format of the run-item being edited (bidsmap[dataformat][datatype][run-item])"""
- self.unknowndatatypes: List[str] = [datatype for datatype in bidsmap.options['unknowntypes'] if datatype in template_bidsmap.dataformat(self.dataformat).datatypes]
- self.ignoredatatypes: List[str] = [datatype for datatype in bidsmap.options['ignoretypes'] if datatype in template_bidsmap.dataformat(self.dataformat).datatypes]
+ self.unknowndatatypes: list[str] = [datatype for datatype in bidsmap.options['unknowntypes'] if datatype in template_bidsmap.dataformat(self.dataformat).datatypes]
+ self.ignoredatatypes: list[str] = [datatype for datatype in bidsmap.options['ignoretypes'] if datatype in template_bidsmap.dataformat(self.dataformat).datatypes]
self.bidsdatatypes = [str(datatype) for datatype in template_bidsmap.dataformat(self.dataformat).datatypes if datatype not in self.unknowndatatypes + self.ignoredatatypes + ['subject', 'session']]
self.bidsignore = bidsmap.options['bidsignore']
self.output_bidsmap = bidsmap
@@ -1842,7 +1842,7 @@ def export_run(self):
bidsmap.save()
QMessageBox.information(self, 'Edit BIDS mapping', f"Successfully exported:\n\n{self.target_run} -> {yamlfile}")
- def get_allowed_suffixes(self) -> Dict[str, set]:
+ def get_allowed_suffixes(self) -> dict[str, set]:
"""Derive the possible suffixes for each datatype from the template. """
allowed_suffixes = {}
@@ -1892,7 +1892,7 @@ def get_help(self):
class CompareWindow(QDialog):
- def __init__(self, runitems: List[RunItem], subid: List[str], sesid: List[str]):
+ def __init__(self, runitems: list[RunItem], subid: list[str], sesid: list[str]):
super().__init__()
# Set up the window
diff --git a/bidscoin/bidsmapper.py b/bidscoin/bidsmapper.py
index ffae54a6..26db48cc 100755
--- a/bidscoin/bidsmapper.py
+++ b/bidscoin/bidsmapper.py
@@ -100,7 +100,7 @@ def bidsmapper(sourcefolder: str, bidsfolder: str, bidsmap: str, template: str,
bidsmap_old = copy.deepcopy(bidsmap_new)
# Import the data scanning plugins
- plugins = [plugin for name in bidsmap_new.plugins if (plugin := bcoin.import_plugin(name, ('bidsmapper_plugin',)))]
+ plugins = [plugin for name in bidsmap_new.plugins if (plugin := bcoin.import_plugin(name))]
if not plugins:
LOGGER.warning(f"The plugins listed in your bidsmap['Options'] did not have a usable `bidsmapper_plugin` function, nothing to do")
LOGGER.info('-------------- FINISHED! ------------')
@@ -132,7 +132,7 @@ def bidsmapper(sourcefolder: str, bidsfolder: str, bidsmap: str, template: str,
for plugin in plugins:
LOGGER.verbose(f"Executing plugin: {Path(plugin.__file__).stem} -> {sesfolder}")
trackusage(Path(plugin.__file__).stem)
- plugin.bidsmapper_plugin(sesfolder, bidsmap_new, bidsmap_old, template)
+ plugin.Interface().bidsmapper(sesfolder, bidsmap_new, bidsmap_old, template)
# Clean-up the temporary unpacked data
if unpacked:
diff --git a/bidscoin/plugins/__init__.py b/bidscoin/plugins/__init__.py
index 5baa012c..a67c43fc 100644
--- a/bidscoin/plugins/__init__.py
+++ b/bidscoin/plugins/__init__.py
@@ -1,17 +1,114 @@
-"""Pre-installed plugins"""
+"""Base classes for the pre-installed plugins"""
import logging
import copy
import pandas as pd
from pathlib import Path
from abc import ABC, abstractmethod
-from typing import List
+from typing import Union
+from bidscoin import is_hidden
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from bidscoin.bids import BidsMap # = Circular import
LOGGER = logging.getLogger(__name__)
+class PluginInterface(ABC):
+ """Base interface class for plugins"""
+
+ def __init__(self):
+ pass
+
+ def test(self, options) -> int:
+ """
+ Performs a plugin test
+
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins[__name__]
+ :return: The errorcode: 0 for successful execution, 1 for general plugin errors, etc
+ """
+
+ LOGGER.info(f"Testing {__name__} is not implemented")
+
+ return 0
+
+ @abstractmethod
+ def has_support(self, file: Path, dataformat: str) -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
+
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The valid/supported dataformat of the sourcefile
+ """
+
+ @abstractmethod
+ def get_attribute(self, dataformat, sourcefile: Path, attribute: str, options: dict) -> Union[str, int, float, list]:
+ """
+ This plugin supports reading attributes from DICOM and PAR dataformats
+
+ :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which the attribute value should be read
+ :param attribute: The attribute key for which the value should be read
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['nibabel2bids']
+ :return: The retrieved attribute value
+ """
+
+ def bidsmapper(self, session: Path, bidsmap_new: 'BidsMap', bidsmap_old: 'BidsMap', template: 'BidsMap') -> None:
+ """
+ The goal of this plugin function is to identify all the different runs in the session and update the
+ bidsmap if a new run is discovered
+
+ :param session: The full-path name of the subject/session raw data source folder
+ :param bidsmap_new: The new study bidsmap that we are building
+ :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
+ :param template: The template bidsmap with the default heuristics
+ """
+
+ # See for every source file in the session if we already discovered it or not
+ for sourcefile in session.rglob('*'):
+
+ # Check if the sourcefile is of a supported dataformat
+ if is_hidden(sourcefile.relative_to(session)) or not (dataformat := self.has_support(sourcefile, dataformat='')):
+ continue
+
+ # See if we can find a matching run in the old bidsmap
+ run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+
+ # If not, see if we can find a matching run in the template
+ if not oldmatch:
+ run, _ = template.get_matching_run(sourcefile, dataformat)
+
+ # See if we have already put the run somewhere in our new bidsmap
+ if not bidsmap_new.exist_run(run):
+
+ # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
+ if not oldmatch:
+ LOGGER.info(f"Discovered sample: {run.datasource}")
+ else:
+ LOGGER.bcdebug(f"Known sample: {run.datasource}")
+
+ # Copy the filled-in run over to the new bidsmap
+ bidsmap_new.insert_run(run)
+
+ else:
+ LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
+
+ @abstractmethod
+ def bidscoiner(self, session: Path, bidsmap: 'BidsMap', bidsses: Path) -> None:
+ """
+ The bidscoiner plugin to convert the session Nibabel source-files into BIDS-valid NIfTI-files in the
+ corresponding bids session-folder
+
+ :param session: The full-path name of the subject/session source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: Nothing (i.e. personal data is not available)
+ """
+
+
class EventsParser(ABC):
- """Parser for stimulus presentation logfiles"""
+ """Base parser for stimulus presentation logfiles"""
def __init__(self, sourcefile: Path, eventsdata: dict, options: dict):
"""
@@ -90,21 +187,21 @@ def eventstable(self) -> pd.DataFrame:
return df.loc[rows.values].sort_values(by='onset')
@property
- def columns(self) -> List[dict]:
+ def columns(self) -> list[dict]:
"""List with mappings for the column names of the eventstable"""
return self._data.get('columns') or []
@columns.setter
- def columns(self, value: List[dict]):
+ def columns(self, value: list[dict]):
self._data['columns'] = value
@property
- def rows(self) -> List[dict]:
+ def rows(self) -> list[dict]:
"""List with fullmatch regular expression dictionaries that yield row sets in the eventstable"""
return self._data.get('rows') or []
@rows.setter
- def rows(self, value: List[dict]):
+ def rows(self, value: list[dict]):
self._data['rows'] = value
@property
@@ -139,8 +236,8 @@ def is_float(s):
LOGGER.warning(f"Second events column must be named 'duration', got '{key}' instead\n{self}")
valid = False
- if len(self.time.get('cols',[])) < 2:
- LOGGER.warning(f"Events table must have at least two timecol items, got {len(self.time.get('cols',[]))} instead\n{self}")
+ if len(self.time.get('cols', [])) < 2:
+ LOGGER.warning(f"Events table must have at least two timecol items, got {len(self.time.get('cols', []))} instead\n{self}")
return False
elif not is_float(self.time.get('unit')):
@@ -150,7 +247,7 @@ def is_float(s):
# Check if the logtable has existing and unique column names
columns = self.logtable.columns
for name in set([name for item in self.columns for name in item.values()] + [name for item in self.rows for name in item['include'].keys()] +
- [*self.time.get('start',{}).keys()] + self.time.get('cols',[])):
+ [*self.time.get('start', {}).keys()] + self.time.get('cols', [])):
if name and name not in columns:
LOGGER.warning(f"Column '{name}' not found in the event table of {self}")
valid = False
@@ -164,4 +261,3 @@ def write(self, targetfile: Path):
"""Write the eventstable to a BIDS events.tsv file"""
self.eventstable.to_csv(targetfile, sep='\t', index=False)
-
diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py
index 8a7d9a58..dbc81a89 100644
--- a/bidscoin/plugins/dcm2niix2bids.py
+++ b/bidscoin/plugins/dcm2niix2bids.py
@@ -11,11 +11,12 @@
import json
import ast
from bids_validator import BIDSValidator
-from typing import Union, List
+from typing import Union
from pathlib import Path
-from bidscoin import bcoin, bids, run_command, lsdirs, due, Doi
+from bidscoin import bids, run_command, lsdirs, due, Doi
from bidscoin.utilities import physio
from bidscoin.bids import BidsMap, DataFormat, Plugin, Plugins
+from bidscoin.plugins import PluginInterface
try:
from nibabel.testing import data_path
except ImportError:
@@ -36,503 +37,504 @@
'fallback': 'y'}) # Appends unhandled dcm2niix suffixes to the `acq` label if 'y' (recommended, else the suffix data is discarding)
-def test(options: Plugin=OPTIONS) -> int:
- """
- Performs shell tests of dcm2niix
+class Interface(PluginInterface):
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['dcm2niix2bids']
- :return: The errorcode (e.g 0 if the tool generated the expected result, > 0 if there was a tool error)
- """
+ def test(self, options: Plugin=OPTIONS) -> int:
+ """
+ Performs shell tests of dcm2niix
- LOGGER.info('Testing the dcm2niix2bids installation:')
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['dcm2niix2bids']
+ :return: The errorcode (e.g 0 if the tool generated the expected result, > 0 if there was a tool error)
+ """
- if not hasattr(physio, 'readphysio') or not hasattr(physio, 'physio2tsv'):
- LOGGER.error("Could not import the expected 'readphysio' and/or 'physio2tsv' from the physio.py utility")
- return 1
- if 'command' not in {**OPTIONS, **options}:
- LOGGER.error(f"The expected 'command' key is not defined in the dcm2niix2bids options")
- return 1
- if 'args' not in {**OPTIONS, **options}:
- LOGGER.warning(f"The expected 'args' key is not defined in the dcm2niix2bids options")
+ LOGGER.info('Testing the dcm2niix2bids installation:')
- # Test the dcm2niix installation
- errorcode = run_command(f"{options.get('command', OPTIONS['command'])} -v", (0,3))
+ if not hasattr(physio, 'readphysio') or not hasattr(physio, 'physio2tsv'):
+ LOGGER.error("Could not import the expected 'readphysio' and/or 'physio2tsv' from the physio.py utility")
+ return 1
+ if 'command' not in {**OPTIONS, **options}:
+ LOGGER.error(f"The expected 'command' key is not defined in the dcm2niix2bids options")
+ return 1
+ if 'args' not in {**OPTIONS, **options}:
+ LOGGER.warning(f"The expected 'args' key is not defined in the dcm2niix2bids options")
- # Test reading an attribute from a PAR-file
- parfile = Path(data_path)/'phantom_EPI_asc_CLEAR_2_1.PAR'
- try:
- assert has_support(parfile) == 'PAR'
- assert get_attribute('PAR', parfile, 'exam_name', options) == 'Konvertertest'
- except Exception as pluginerror:
- LOGGER.error(f"Could not read attribute(s) from {parfile}:\n{pluginerror}")
- return 1
+ # Test the dcm2niix installation
+ errorcode = run_command(f"{options.get('command', OPTIONS['command'])} -v", (0,3))
+
+ # Test reading an attribute from a PAR-file
+ parfile = Path(data_path)/'phantom_EPI_asc_CLEAR_2_1.PAR'
+ try:
+ assert self.has_support(parfile) == 'PAR'
+ assert self.get_attribute('PAR', parfile, 'exam_name', options) == 'Konvertertest'
+ except Exception as pluginerror:
+ LOGGER.error(f"Could not read attribute(s) from {parfile}:\n{pluginerror}")
+ return 1
- return errorcode if errorcode != 3 else 0
+ return errorcode if errorcode != 3 else 0
+ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
-def has_support(file: Path, dataformat: Union[DataFormat, str]='') -> str:
- """
- This plugin function assesses whether a sourcefile is of a supported dataformat
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The valid/supported dataformat of the sourcefile
+ """
- :param file: The sourcefile that is assessed
- :param dataformat: The requested dataformat (optional requirement)
- :return: The valid/supported dataformat of the sourcefile
- """
+ if dataformat and dataformat not in ('DICOM', 'PAR'):
+ return ''
+
+ if bids.is_dicomfile(file): # To support pet2bids add: and bids.get_dicomfield('Modality', file) != 'PT'
+ return 'DICOM'
+
+ if bids.is_parfile(file):
+ return 'PAR'
- if dataformat and dataformat not in ('DICOM', 'PAR'):
return ''
- if bids.is_dicomfile(file): # To support pet2bids add: and bids.get_dicomfield('Modality', file) != 'PT'
- return 'DICOM'
-
- if bids.is_parfile(file):
- return 'PAR'
-
- return ''
-
-
-def get_attribute(dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options) -> Union[str, int]:
- """
- This plugin supports reading attributes from DICOM and PAR dataformats
-
- :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
- :param sourcefile: The sourcefile from which the attribute value should be read
- :param attribute: The attribute key for which the value should be read
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['dcm2niix2bids']
- :return: The retrieved attribute value
- """
- if dataformat == 'DICOM':
- return bids.get_dicomfield(attribute, sourcefile)
-
- if dataformat == 'PAR':
- return bids.get_parfield(attribute, sourcefile)
-
-
-def bidsmapper_plugin(session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
- """
- The goal of this plugin function is to identify all the different runs in the session and update the
- bidsmap if a new run is discovered
-
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap_new: The new study bidsmap that we are building
- :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
- :param template: The template bidsmap with the default heuristics
- """
-
- # Get started
- plugins = Plugins({'dcm2niix2bids': bidsmap_new.plugins['dcm2niix2bids']})
- datasource = bids.get_datasource(session, plugins)
- dataformat = datasource.dataformat
- if not dataformat:
- return
-
- # Collect the different DICOM/PAR source files for all runs in the session
- sourcefiles: List[Path] = []
- if dataformat == 'DICOM':
- for sourcedir in lsdirs(session, '**/*'):
- for n in range(1): # Option: Use range(2) to scan two files and catch e.g. magnitude1/2 fieldmap files that are stored in one Series folder (but bidscoiner sees only the first file anyhow and it makes bidsmapper 2x slower :-()
- sourcefile = bids.get_dicomfile(sourcedir, n)
- if sourcefile.name:
- sourcefiles.append(sourcefile)
- elif dataformat == 'PAR':
- sourcefiles = bids.get_parfiles(session)
- else:
- LOGGER.error(f"Unsupported dataformat '{dataformat}'")
-
- # See for every data source in the session if we already discovered it or not
- for sourcefile in sourcefiles:
-
- # Check if the source files all have approximately the same size (difference < 50kB)
- if dataformat == 'DICOM':
- for file in sourcefile.parent.iterdir():
- if abs(file.stat().st_size - sourcefile.stat().st_size) > 1024 * 50 and file.suffix == sourcefile.suffix:
- LOGGER.warning(f"Not all {file.suffix}-files in '{sourcefile.parent}' have the same size. This may be OK but can also be indicative of a truncated acquisition or file corruption(s)")
- break
- # See if we can find a matching run in the old bidsmap
- run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options) -> Union[str, int]:
+ """
+ This plugin supports reading attributes from DICOM and PAR dataformats
- # If not, see if we can find a matching run in the template
- if not oldmatch:
- LOGGER.bcdebug('No match found in the study bidsmap, now trying the template bidsmap')
- run, _ = template.get_matching_run(sourcefile, dataformat)
+ :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which the attribute value should be read
+ :param attribute: The attribute key for which the value should be read
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['dcm2niix2bids']
+ :return: The retrieved attribute value
+ """
+ if dataformat == 'DICOM':
+ return bids.get_dicomfield(attribute, sourcefile)
- # See if we have already put the run somewhere in our new bidsmap
- if not bidsmap_new.exist_run(run):
+ if dataformat == 'PAR':
+ return bids.get_parfield(attribute, sourcefile)
- # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
- if not oldmatch:
- LOGGER.info(f"Discovered sample: {run.datasource}")
-
- # Try to automagically set the {part: phase/imag/real} (should work for Siemens data)
- if not run.datatype == '' and 'part' in run.bids and not run.bids['part'][-1] and run.attributes.get('ImageType'): # part[-1]==0 -> part is not specified
- imagetype = ast.literal_eval(run.attributes['ImageType']) # E.g. ImageType = "['ORIGINAL', 'PRIMARY', 'M', 'ND']"
- if 'P' in imagetype:
- run.bids['part'][-1] = run.bids['part'].index('phase') # E.g. part = ['', mag, phase, real, imag, 0]
- # elif 'M' in imagetype:
- # run.bids['part'][-1] = run.bids['part'].index('mag')
- elif 'I' in imagetype:
- run.bids['part'][-1] = run.bids['part'].index('imag')
- elif 'R' in imagetype:
- run.bids['part'][-1] = run.bids['part'].index('real')
- if run.bids['part'][-1]:
- LOGGER.verbose(f"Updated {run} entity: 'part' -> '{run.bids['part'][run.bids['part'][-1]]}' ({imagetype})")
+ def bidsmapper(self, session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
+ """
+ The goal of this plugin function is to identify all the different runs in the session and update the
+ bidsmap if a new run is discovered
- else:
- LOGGER.bcdebug(f"Known sample: {run.datasource}")
+ :param session: The full-path name of the subject/session raw data source folder
+ :param bidsmap_new: The new study bidsmap that we are building
+ :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
+ :param template: The template bidsmap with the default heuristics
+ """
- # Copy the filled-in run over to the new bidsmap
- bidsmap_new.insert_run(run)
+ # Get started
+ plugins = Plugins({'dcm2niix2bids': bidsmap_new.plugins['dcm2niix2bids']})
+ datasource = bids.get_datasource(session, plugins)
+ dataformat = datasource.dataformat
+ if not dataformat:
+ return
+ # Collect the different DICOM/PAR source files for all runs in the session
+ sourcefiles: list[Path] = []
+ if dataformat == 'DICOM':
+ for sourcedir in lsdirs(session, '**/*'):
+ for n in range(1): # Option: Use range(2) to scan two files and catch e.g. magnitude1/2 fieldmap files that are stored in one Series folder (but bidscoiner sees only the first file anyhow and it makes bidsmapper 2x slower :-()
+ sourcefile = bids.get_dicomfile(sourcedir, n)
+ if sourcefile.name:
+ sourcefiles.append(sourcefile)
+ elif dataformat == 'PAR':
+ sourcefiles = bids.get_parfiles(session)
else:
- LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
-
-
-@due.dcite(Doi('10.1016/j.jneumeth.2016.03.001'), description='dcm2niix: DICOM to NIfTI converter', tags=['reference-implementation'])
-def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
- """
- The bidscoiner plugin to convert the session DICOM and PAR/REC source-files into BIDS-valid NIfTI-files in the corresponding
- bids session-folder and extract personals (e.g. Age, Sex) from the source header. The bidsmap options for this plugin can be found in:
-
- bidsmap.plugins['spec2nii2bids']
-
- :param session: The full-path name of the subject/session source folder
- :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
- :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
- :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
- """
-
- # Get the subject identifiers and the BIDS root folder from the bidsses folder
- subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
- sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
- bidsfolder = bidsses.parent.parent if sesid else bidsses.parent
-
- # Get started and see what dataformat we have
- options = Plugin(bidsmap.plugins['dcm2niix2bids'])
- bidsignore: list = bidsmap.options['bidsignore']
- fallback = 'fallback' if options.get('fallback','y').lower() in ('y', 'yes', 'true') else ''
- datasource = bids.get_datasource(session, Plugins({'dcm2niix2bids': options}))
- dataformat = datasource.dataformat
- if not dataformat:
- LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
- return
-
- # Make a list of all the data sources/runs
- sources: List[Path] = []
- manufacturer = 'UNKNOWN'
- if dataformat == 'DICOM':
- sources = lsdirs(session, '**/*')
- manufacturer = datasource.attributes('Manufacturer')
- elif dataformat == 'PAR':
- sources = bids.get_parfiles(session)
- manufacturer = 'Philips Medical Systems'
- else:
- LOGGER.error(f"Unsupported dataformat '{dataformat}'")
-
- # Read or create a scans_table and tsv-file
- scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
- if scans_tsv.is_file():
- scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
- else:
- scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
- scans_table.index.name = 'filename'
-
- # Process all the source files / folders
- for source in sources:
-
- # Get a sourcefile
+ LOGGER.error(f"Unsupported dataformat '{dataformat}'")
+
+ # See for every data source in the session if we already discovered it or not
+ for sourcefile in sourcefiles:
+
+ # Check if the source files all have approximately the same size (difference < 50kB)
+ if dataformat == 'DICOM':
+ for file in sourcefile.parent.iterdir():
+ if abs(file.stat().st_size - sourcefile.stat().st_size) > 1024 * 50 and file.suffix == sourcefile.suffix:
+ LOGGER.warning(f"Not all {file.suffix}-files in '{sourcefile.parent}' have the same size. This may be OK but can also be indicative of a truncated acquisition or file corruption(s)")
+ break
+
+ # See if we can find a matching run in the old bidsmap
+ run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+
+ # If not, see if we can find a matching run in the template
+ if not oldmatch:
+ LOGGER.bcdebug('No match found in the study bidsmap, now trying the template bidsmap')
+ run, _ = template.get_matching_run(sourcefile, dataformat)
+
+ # See if we have already put the run somewhere in our new bidsmap
+ if not bidsmap_new.exist_run(run):
+
+ # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
+ if not oldmatch:
+
+ LOGGER.info(f"Discovered sample: {run.datasource}")
+
+ # Try to automagically set the {part: phase/imag/real} (should work for Siemens data)
+ if not run.datatype == '' and 'part' in run.bids and not run.bids['part'][-1] and run.attributes.get('ImageType'): # part[-1]==0 -> part is not specified
+ imagetype = ast.literal_eval(run.attributes['ImageType']) # E.g. ImageType = "['ORIGINAL', 'PRIMARY', 'M', 'ND']"
+ if 'P' in imagetype:
+ run.bids['part'][-1] = run.bids['part'].index('phase') # E.g. part = ['', mag, phase, real, imag, 0]
+ # elif 'M' in imagetype:
+ # run.bids['part'][-1] = run.bids['part'].index('mag')
+ elif 'I' in imagetype:
+ run.bids['part'][-1] = run.bids['part'].index('imag')
+ elif 'R' in imagetype:
+ run.bids['part'][-1] = run.bids['part'].index('real')
+ if run.bids['part'][-1]:
+ LOGGER.verbose(f"Updated {run} entity: 'part' -> '{run.bids['part'][run.bids['part'][-1]]}' ({imagetype})")
+
+ else:
+ LOGGER.bcdebug(f"Known sample: {run.datasource}")
+
+ # Copy the filled-in run over to the new bidsmap
+ bidsmap_new.insert_run(run)
+
+ else:
+ LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
+
+
+ @due.dcite(Doi('10.1016/j.jneumeth.2016.03.001'), description='dcm2niix: DICOM to NIfTI converter', tags=['reference-implementation'])
+ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
+ """
+ The bidscoiner plugin to convert the session DICOM and PAR/REC source-files into BIDS-valid NIfTI-files in the corresponding
+ bids session-folder and extract personals (e.g. Age, Sex) from the source header. The bidsmap options for this plugin can be found in:
+
+ bidsmap.plugins['spec2nii2bids']
+
+ :param session: The full-path name of the subject/session source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
+ """
+
+ # Get the subject identifiers and the BIDS root folder from the bidsses folder
+ subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
+ sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
+ bidsfolder = bidsses.parent.parent if sesid else bidsses.parent
+
+ # Get started and see what dataformat we have
+ options = Plugin(bidsmap.plugins['dcm2niix2bids'])
+ bidsignore: list = bidsmap.options['bidsignore']
+ fallback = 'fallback' if options.get('fallback','y').lower() in ('y', 'yes', 'true') else ''
+ datasource = bids.get_datasource(session, Plugins({'dcm2niix2bids': options}))
+ dataformat = datasource.dataformat
+ if not dataformat:
+ LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
+ return
+
+ # Make a list of all the data sources/runs
+ sources: list[Path] = []
+ manufacturer = 'UNKNOWN'
if dataformat == 'DICOM':
- sourcefile = bids.get_dicomfile(source)
+ sources = lsdirs(session, '**/*')
+ manufacturer = datasource.attributes('Manufacturer')
+ elif dataformat == 'PAR':
+ sources = bids.get_parfiles(session)
+ manufacturer = 'Philips Medical Systems'
else:
- sourcefile = source
- if not sourcefile.name or not has_support(sourcefile):
- continue
-
- # Get a matching run from the bidsmap
- run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
-
- # Check if we should ignore this run
- if run.datatype in bidsmap.options['ignoretypes']:
- LOGGER.info(f"--> Leaving out: {run.datasource}")
- bids.bidsprov(bidsses, source, run) # Write out empty provenance logging data
- continue
-
- # Check if we already know this run
- if not runid:
- LOGGER.error(f"--> Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
- bids.bidsprov(bidsses, source) # Write out empty provenance logging data
- continue
-
- LOGGER.info(f"--> Coining: {run.datasource}")
-
- # Create the BIDS session/datatype output folder
- suffix = run.datasource.dynamicvalue(run.bids['suffix'], True, True)
- outfolder = bidsses/run.datatype
- outfolder.mkdir(parents=True, exist_ok=True)
-
- # Compose the BIDS filename using the matched run
- ignore = bids.check_ignore(run.datatype, bidsignore)
- bidsname = run.bidsname(subid, sesid, not ignore, runtime=True)
- ignore = ignore or bids.check_ignore(bidsname+'.json', bidsignore, 'file')
- runindex = bids.get_bidsvalue(bidsname, 'run')
- bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
- targets = set() # -> A store for all fullpath output targets (.nii/.tsv) for this bidsname
-
- # Check if the bidsname is valid
- bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
- isbids = BIDSValidator().is_bids(bidstest)
- if not isbids and not ignore:
- LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
-
- # Check if the output file already exists (-> e.g. when a static runindex is used)
- if (outfolder/bidsname).with_suffix('.json').is_file():
- LOGGER.warning(f"{outfolder/bidsname}.* already exists and will be deleted -- check your results carefully!")
- for ext in ('.nii.gz', '.nii', '.json', '.tsv', '.tsv.gz', '.bval', '.bvec'):
- (outfolder/bidsname).with_suffix(ext).unlink(missing_ok=True)
-
- # Check if the source files all have approximately the same size (difference < 50kB)
- for file in source.iterdir() if source.is_dir() else []:
- if abs(file.stat().st_size - sourcefile.stat().st_size) > 1024 * 50 and file.suffix == sourcefile.suffix:
- LOGGER.warning(f"Not all {file.suffix}-files in '{source}' have the same size. This may be OK but can also be indicative of a truncated acquisition or file corruption(s)")
- break
-
- # Convert physiological log files (dcm2niix can't handle these)
- if suffix == 'physio':
- target = (outfolder/bidsname).with_suffix('.tsv.gz')
- if bids.get_dicomfile(source, 2).name: # TODO: issue warning or support PAR
- LOGGER.warning(f"Found > 1 DICOM file in {source}, using: {sourcefile}")
- try:
- physiodata = physio.readphysio(sourcefile)
- physio.physio2tsv(physiodata, target)
- if target.is_file():
- targets.add(target)
- except Exception as physioerror:
- LOGGER.error(f"Could not read/convert physiological file: {sourcefile}\n{physioerror}")
- continue
+ LOGGER.error(f"Unsupported dataformat '{dataformat}'")
- # Convert the source-files in the run folder to NIfTI's in the BIDS-folder
+ # Read or create a scans_table and tsv-file
+ scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
+ if scans_tsv.is_file():
+ scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
else:
- command = '{command} {args} -f "{filename}" -o "{outfolder}" "{source}"'.format(
- command = options['command'],
- args = options.get('args',''),
- filename = bidsname,
- outfolder = outfolder,
- source = source)
- if run_command(command) and not next(outfolder.glob(f"{bidsname}*"), None):
+ scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
+ scans_table.index.name = 'filename'
+
+ # Process all the source files / folders
+ for source in sources:
+
+ # Get a sourcefile
+ if dataformat == 'DICOM':
+ sourcefile = bids.get_dicomfile(source)
+ else:
+ sourcefile = source
+ if not sourcefile.name or not self.has_support(sourcefile):
continue
- # Collect the bidsname
- target = next(outfolder.glob(f"{bidsname}.nii*"), None)
- if target:
- targets.add(target)
-
- # Handle the ABCD GE pepolar sequence
- extrafile = next(outfolder.glob(f"{bidsname}a.nii*"), None)
- if extrafile:
- # Load the json meta-data to see if it's a pepolar sequence
- with extrafile.with_suffix('').with_suffix('.json').open('r') as json_fid:
- jsondata = json.load(json_fid)
- if 'PhaseEncodingPolarityGE' in jsondata:
- ext = ''.join(extrafile.suffixes)
- invfile = bids.get_bidsvalue(outfolder/(bidsname+ext), 'dir', bids.get_bidsvalue(bidsname,'dir') + jsondata['PhaseEncodingPolarityGE'])
- LOGGER.verbose(f"Renaming GE reversed polarity image: {extrafile} -> {invfile}")
- extrafile.replace(invfile)
- extrafile.with_suffix('').with_suffix('.json').replace(invfile.with_suffix('').with_suffix('.json'))
- targets.add(invfile)
- else:
- LOGGER.error(f"Unexpected variants of {outfolder/bidsname}* were produced by dcm2niix. Possibly this can be remedied by using the dcm2niix -i option (to ignore derived, localizer and 2D images) or by clearing the BIDS folder before running bidscoiner")
+ # Get a matching run from the bidsmap
+ run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
- # Replace uncropped output image with the cropped one. TODO: fix
- if '-x y' in options.get('args',''):
- for dcm2niixfile in sorted(outfolder.glob(bidsname + '*_Crop_*')): # e.g. *_Crop_1.nii.gz
- ext = ''.join(dcm2niixfile.suffixes)
- newbidsfile = str(dcm2niixfile).rsplit(ext,1)[0].rsplit('_Crop_',1)[0] + ext
- LOGGER.verbose(f"Found dcm2niix _Crop_ postfix, replacing original file\n{dcm2niixfile} ->\n{newbidsfile}")
- dcm2niixfile.replace(newbidsfile)
+ # Check if we should ignore this run
+ if run.datatype in bidsmap.options['ignoretypes']:
+ LOGGER.info(f"--> Leaving out: {run.datasource}")
+ bids.bidsprov(bidsses, source, run) # Write out empty provenance logging data
+ continue
- # Check if there are files that got additional postfixes from dcm2niix. See: https://github.com/rordenlab/dcm2niix/blob/master/FILENAMING.md
- dcm2niixpostfixes = ('_c', '_i', '_Eq', '_real', '_imaginary', '_MoCo', '_t', '_Tilt', '_e', '_ph', '_ADC', '_fieldmaphz') #_c%d, _e%d and _ph (and any combination of these in that order) are for multi-coil data, multi-echo data and phase data
- dcm2niixfiles = sorted(set(dcm2niixfile for dcm2niixpostfix in dcm2niixpostfixes for dcm2niixfile in outfolder.glob(f"{bidsname}*{dcm2niixpostfix}*.nii*")))
- dcm2niixfiles = [dcm2niixfile for dcm2niixfile in dcm2niixfiles if not (re.match(r'sub-.*_echo-[0-9]*\.nii', dcm2niixfile.name) or
- re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))] # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see GitHub issue #232)
-
- # Rename all dcm2niix files that got additional postfixes (i.e. store the postfixes in the bidsname)
- for dcm2niixfile in dcm2niixfiles:
-
- # Strip each dcm2niix postfix and assign it to the proper bids entity in the new bidsname (else assign it to the fallback entity)
- ext = ''.join(dcm2niixfile.suffixes)
- postfixes = dcm2niixfile.name.split(bidsname)[1].rsplit(ext)[0].split('_')[1:]
- newbidsname = bids.insert_bidskeyval(dcm2niixfile.name, 'run', runindex, ignore) # Restart the run-index. NB: Unlike bidsname, newbidsname has a file extension
- for postfix in postfixes:
-
- # Patch the echo entity in the newbidsname with the dcm2niix echo info # NB: We can't rely on the bids-entity info here because manufacturers can e.g. put multiple echos in one series/run-folder
- if 'echo' in run.bids and postfix.startswith('e'):
- echonr = f"_{postfix}".replace('_e','') # E.g. postfix='e1'
- if not echonr:
- echonr = '1'
- if echonr.isdecimal():
- newbidsname = bids.insert_bidskeyval(newbidsname, 'echo', echonr.lstrip('0'), ignore) # In contrast to other labels, run and echo labels MUST be integers. Those labels MAY include zero padding, but this is NOT RECOMMENDED to maintain their uniqueness
- elif echonr[0:-1].isdecimal():
- LOGGER.verbose(f"Splitting off echo-number {echonr[0:-1]} from the '{postfix}' postfix")
- newbidsname = bids.insert_bidskeyval(newbidsname, 'echo', echonr[0:-1].lstrip('0'), ignore) # Strip of the 'a', 'b', etc. from `e1a`, `e1b`, etc
- newbidsname = bids.get_bidsvalue(newbidsname, fallback, echonr[-1]) # Append the 'a' to the fallback-label
- else:
- LOGGER.error(f"Unexpected postix '{postfix}' found in {dcm2niixfile}")
- newbidsname = bids.get_bidsvalue(newbidsname, fallback, postfix) # Append the unknown postfix to the fallback-label
-
- # Patch the phase entity in the newbidsname with the dcm2niix mag/phase info
- elif 'part' in run.bids and postfix in ('ph','real','imaginary'): # e.g. part: ['', 'mag', 'phase', 'real', 'imag', 0]
- if postfix == 'ph':
- newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'phase', ignore)
- if postfix == 'real':
- newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'real', ignore)
- if postfix == 'imaginary':
- newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'imag', ignore)
-
- # Patch fieldmap images (NB: datatype=='fmap' is too broad, see the fmap.yaml file)
- elif suffix in bids.filerules.fmap.fieldmaps.suffixes: # i.e. in ('magnitude','magnitude1','magnitude2','phase1','phase2','phasediff','fieldmap')
- if len(dcm2niixfiles) not in (1, 2, 3, 4): # Phase/echo data may be stored in the same data source/run folder
- LOGGER.verbose(f"Unknown fieldmap {outfolder/bidsname} for '{postfix}'")
- newbidsname = newbidsname.replace('_magnitude1a', '_magnitude2') # First catch this potential weird/rare case
- newbidsname = newbidsname.replace('_magnitude1_pha', '_phase2') # First catch this potential weird/rare case
- newbidsname = newbidsname.replace('_magnitude1_e1', '_magnitude1') # Case 2 = Two phase and magnitude images
- newbidsname = newbidsname.replace('_magnitude1_e2', '_magnitude2') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
- newbidsname = newbidsname.replace('_magnitude2_e1', '_magnitude1') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
- newbidsname = newbidsname.replace('_magnitude2_e2', '_magnitude2') # Case 2
- if len(dcm2niixfiles) in (2,3): # Case 1 = One or two magnitude + one phasediff image
- newbidsname = newbidsname.replace('_magnitude1_ph', '_phasediff')
- newbidsname = newbidsname.replace('_magnitude2_ph', '_phasediff')
- newbidsname = newbidsname.replace('_phasediff_e1', '_phasediff') # Case 1
- newbidsname = newbidsname.replace('_phasediff_e2', '_phasediff') # Case 1
- newbidsname = newbidsname.replace('_phasediff_ph', '_phasediff') # Case 1
- newbidsname = newbidsname.replace('_magnitude1_ph', '_phase1') # Case 2: One or two magnitude and phase images in one folder
- newbidsname = newbidsname.replace('_magnitude2_ph', '_phase2') # Case 2: Two magnitude + two phase images in one folder
- newbidsname = newbidsname.replace('_phase1_e1', '_phase1') # Case 2
- newbidsname = newbidsname.replace('_phase1_e2', '_phase2') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
- newbidsname = newbidsname.replace('_phase2_e1', '_phase1') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
- newbidsname = newbidsname.replace('_phase2_e2', '_phase2') # Case 2
- newbidsname = newbidsname.replace('_phase1_ph', '_phase1') # Case 2: One or two magnitude and phase images in one folder
- newbidsname = newbidsname.replace('_phase2_ph', '_phase2') # Case 2: Two magnitude + two phase images in one folder
- newbidsname = newbidsname.replace('_magnitude_e1', '_magnitude') # Case 3 = One magnitude + one fieldmap image
- if len(dcm2niixfiles) == 2:
- newbidsname = newbidsname.replace('_fieldmap_e1', '_magnitude') # Case 3: One magnitude + one fieldmap image in one folder
- newbidsname = newbidsname.replace('_magnitude_fieldmaphz', '_fieldmap')
- newbidsname = newbidsname.replace('_fieldmap_fieldmaphz', '_fieldmap')
- newbidsname = newbidsname.replace('_fieldmap_e1', '_fieldmap') # Case 3
- newbidsname = newbidsname.replace('_magnitude_ph', '_fieldmap') # Case 3: One magnitude + one fieldmap image in one folder
- newbidsname = newbidsname.replace('_fieldmap_ph', '_fieldmap') # Case 3
-
- # Append the dcm2niix info to the fallback-label, may need to be improved/elaborated for future BIDS standards, supporting multi-coil data
- else:
- newbidsname = bids.get_bidsvalue(newbidsname, fallback, postfix)
-
- # Remove the added postfix from the new bidsname
- newbidsname = newbidsname.replace(f"_{postfix}_",'_') # If it is not last
- newbidsname = newbidsname.replace(f"_{postfix}.",'.') # If it is last
-
- # The ADC images are not BIDS compliant
- if postfix == 'ADC':
- LOGGER.warning(f"The {newbidsname} image is a derivate / not BIDS-compliant -- you can probably delete it safely and update {scans_tsv}")
-
- # Save the NIfTI file with the newly constructed name
- newbidsname = run.increment_runindex(outfolder, newbidsname, scans_table, targets) # Update the runindex now that the name has changed
- newbidsfile = outfolder/newbidsname
- LOGGER.verbose(f"Found dcm2niix {postfixes} postfixes, renaming\n{dcm2niixfile} ->\n{newbidsfile}")
- if newbidsfile.is_file():
- LOGGER.warning(f"Overwriting existing {newbidsfile} file -- check your results carefully!")
- dcm2niixfile.replace(newbidsfile)
- targets.add(newbidsfile)
-
- # Rename all associated files (i.e. the json-, bval- and bvec-files)
- for oldfile in outfolder.glob(dcm2niixfile.with_suffix('').stem + '.*'):
- oldfile.replace(newbidsfile.with_suffix('').with_suffix(''.join(oldfile.suffixes)))
-
- # Write out provenance data
- bids.bidsprov(bidsses, source, run, targets)
-
- # Loop over all targets (i.e. the produced output files) and edit the json sidecar data
- for target in sorted(targets):
-
- # Load/copy over the source meta-data
- jsonfile = target.with_suffix('').with_suffix('.json')
- if not jsonfile.is_file():
- LOGGER.warning(f"Unexpected conversion result, could not find: {jsonfile}")
- metadata = bids.poolmetadata(run.datasource, jsonfile, run.meta, options.get('meta',[]))
-
- # Remove the bval/bvec files of sbref- and inv-images (produced by dcm2niix but not allowed by the BIDS specifications)
- if ((run.datatype == 'dwi' and suffix == 'sbref') or
- (run.datatype == 'fmap' and suffix == 'epi') or
- (run.datatype == 'anat' and suffix == 'MP2RAGE')):
- for ext in ('.bval', '.bvec'):
- bfile = target.with_suffix('').with_suffix(ext)
- if bfile.is_file() and not bids.check_ignore(bfile.name, bidsignore, 'file'):
- bdata = pd.read_csv(bfile, header=None)
- if bdata.any(axis=None):
- LOGGER.warning(f"Found unexpected non-zero b-values in '{bfile}'")
- LOGGER.verbose(f"Removing BIDS-invalid b0-file: {bfile} -> {jsonfile}")
- metadata[ext[1:]] = bdata.values.tolist()
- bfile.unlink()
-
- # Save the meta-data to the json sidecar-file
- if metadata:
- with jsonfile.open('w') as json_fid:
- json.dump(metadata, json_fid, indent=4)
-
- # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
- if not ignore:
- acq_time = ''
- if dataformat == 'DICOM':
- acq_time = f"{run.datasource.attributes('AcquisitionDate')}T{run.datasource.attributes('AcquisitionTime')}"
- elif dataformat == 'PAR':
- acq_time = run.datasource.attributes('exam_date')
- if not acq_time or acq_time == 'T':
- acq_time = f"1925-01-01T{metadata.get('AcquisitionTime','')}"
+ # Check if we already know this run
+ if not runid:
+ LOGGER.error(f"--> Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
+ bids.bidsprov(bidsses, source) # Write out empty provenance logging data
+ continue
+
+ LOGGER.info(f"--> Coining: {run.datasource}")
+
+ # Create the BIDS session/datatype output folder
+ suffix = run.datasource.dynamicvalue(run.bids['suffix'], True, True)
+ outfolder = bidsses/run.datatype
+ outfolder.mkdir(parents=True, exist_ok=True)
+
+ # Compose the BIDS filename using the matched run
+ ignore = bids.check_ignore(run.datatype, bidsignore)
+ bidsname = run.bidsname(subid, sesid, not ignore, runtime=True)
+ ignore = ignore or bids.check_ignore(bidsname+'.json', bidsignore, 'file')
+ runindex = bids.get_bidsvalue(bidsname, 'run')
+ bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
+ targets = set() # -> A store for all fullpath output targets (.nii/.tsv) for this bidsname
+
+ # Check if the bidsname is valid
+ bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
+ isbids = BIDSValidator().is_bids(bidstest)
+ if not isbids and not ignore:
+ LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
+
+ # Check if the output file already exists (-> e.g. when a static runindex is used)
+ if (outfolder/bidsname).with_suffix('.json').is_file():
+ LOGGER.warning(f"{outfolder/bidsname}.* already exists and will be deleted -- check your results carefully!")
+ for ext in ('.nii.gz', '.nii', '.json', '.tsv', '.tsv.gz', '.bval', '.bvec'):
+ (outfolder/bidsname).with_suffix(ext).unlink(missing_ok=True)
+
+ # Check if the source files all have approximately the same size (difference < 50kB)
+ for file in source.iterdir() if source.is_dir() else []:
+ if abs(file.stat().st_size - sourcefile.stat().st_size) > 1024 * 50 and file.suffix == sourcefile.suffix:
+ LOGGER.warning(f"Not all {file.suffix}-files in '{source}' have the same size. This may be OK but can also be indicative of a truncated acquisition or file corruption(s)")
+ break
+
+ # Convert physiological log files (dcm2niix can't handle these)
+ if suffix == 'physio':
+ target = (outfolder/bidsname).with_suffix('.tsv.gz')
+ if bids.get_dicomfile(source, 2).name: # TODO: issue warning or support PAR
+ LOGGER.warning(f"Found > 1 DICOM file in {source}, using: {sourcefile}")
try:
- acq_time = dateutil.parser.parse(acq_time)
- if options.get('anon','y') in ('y','yes'):
- acq_time = acq_time.replace(year=1925, month=1, day=1) # Privacy protection (see BIDS specification)
- acq_time = acq_time.isoformat()
- except Exception as jsonerror:
- LOGGER.warning(f"Could not parse the acquisition time from: {sourcefile}\n{jsonerror}")
- acq_time = 'n/a'
- scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time
-
- # Check if the target output aligns with dcm2niix's "BidsGuess" datatype and filename entities
- if not ignore:
- typeguess, targetguess = metadata.get('BidsGuess') or ['', ''] # BidsGuess: [datatype, filename]
- LOGGER.bcdebug(f"BidsGuess: [{typeguess}, {targetguess}]")
- if typeguess and run.datatype != typeguess:
- LOGGER.info(f"The datatype of {target.relative_to(bidsses)} does not match with the datatype guessed by dcm2niix: {typeguess}")
- elif targetguess and bids.get_bidsvalue(target, 'suffix') != bids.get_bidsvalue(targetguess, 'suffix'):
- LOGGER.info(f"The suffix of {target.relative_to(bidsses)} does not match with the suffix guessed by dcm2niix: {targetguess}")
- for entity in ('part', 'inv', 'echo', 'dir'):
- targetvalue = bids.get_bidsvalue(target, entity)
- guessvalue = bids.get_bidsvalue(targetguess, entity)
- if targetvalue and guessvalue and targetvalue != guessvalue:
- LOGGER.warning(f"The '{entity}_{targetvalue}' value in {target.relative_to(bidsses)} does not match with the '{entity}_{guessvalue}' value guessed by dcm2niix: {targetguess}")
-
- # Write the scans_table to disk
- LOGGER.verbose(f"Writing acquisition time data to: {scans_tsv}")
- scans_table.sort_values(by=['acq_time','filename'], inplace=True)
- scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
-
- # Collect personal data for the participants.tsv file
- if dataformat == 'DICOM': # PAR does not contain personal info?
- age = datasource.attributes('PatientAge') # A string of characters with one of the following formats: nnnD, nnnW, nnnM, nnnY
- try:
- if age.endswith('D'): age = float(age.rstrip('D')) / 365.2524
- elif age.endswith('W'): age = float(age.rstrip('W')) / 52.1775
- elif age.endswith('M'): age = float(age.rstrip('M')) / 12
- elif age.endswith('Y'): age = float(age.rstrip('Y'))
- if age and options.get('anon','y') in ('y','yes'):
- age = int(float(age))
- except Exception as exc:
- LOGGER.warning(f"Could not parse age from: {datasource}\n{exc}")
- personals = {}
- personals['age'] = str(age)
- personals['sex'] = datasource.attributes('PatientSex')
- personals['size'] = datasource.attributes('PatientSize')
- personals['weight'] = datasource.attributes('PatientWeight')
-
- return personals
+ physiodata = physio.readphysio(sourcefile)
+ physio.physio2tsv(physiodata, target)
+ if target.is_file():
+ targets.add(target)
+ except Exception as physioerror:
+ LOGGER.error(f"Could not read/convert physiological file: {sourcefile}\n{physioerror}")
+ continue
+
+ # Convert the source-files in the run folder to NIfTI's in the BIDS-folder
+ else:
+ command = '{command} {args} -f "{filename}" -o "{outfolder}" "{source}"'.format(
+ command = options['command'],
+ args = options.get('args',''),
+ filename = bidsname,
+ outfolder = outfolder,
+ source = source)
+ if run_command(command) and not next(outfolder.glob(f"{bidsname}*"), None):
+ continue
+
+ # Collect the bidsname
+ target = next(outfolder.glob(f"{bidsname}.nii*"), None)
+ if target:
+ targets.add(target)
+
+ # Handle the ABCD GE pepolar sequence
+ extrafile = next(outfolder.glob(f"{bidsname}a.nii*"), None)
+ if extrafile:
+ # Load the json meta-data to see if it's a pepolar sequence
+ with extrafile.with_suffix('').with_suffix('.json').open('r') as json_fid:
+ jsondata = json.load(json_fid)
+ if 'PhaseEncodingPolarityGE' in jsondata:
+ ext = ''.join(extrafile.suffixes)
+ invfile = bids.get_bidsvalue(outfolder/(bidsname+ext), 'dir', bids.get_bidsvalue(bidsname,'dir') + jsondata['PhaseEncodingPolarityGE'])
+ LOGGER.verbose(f"Renaming GE reversed polarity image: {extrafile} -> {invfile}")
+ extrafile.replace(invfile)
+ extrafile.with_suffix('').with_suffix('.json').replace(invfile.with_suffix('').with_suffix('.json'))
+ targets.add(invfile)
+ else:
+ LOGGER.error(f"Unexpected variants of {outfolder/bidsname}* were produced by dcm2niix. Possibly this can be remedied by using the dcm2niix -i option (to ignore derived, localizer and 2D images) or by clearing the BIDS folder before running bidscoiner")
+
+ # Replace uncropped output image with the cropped one. TODO: fix
+ if '-x y' in options.get('args',''):
+ for dcm2niixfile in sorted(outfolder.glob(bidsname + '*_Crop_*')): # e.g. *_Crop_1.nii.gz
+ ext = ''.join(dcm2niixfile.suffixes)
+ newbidsfile = str(dcm2niixfile).rsplit(ext,1)[0].rsplit('_Crop_',1)[0] + ext
+ LOGGER.verbose(f"Found dcm2niix _Crop_ postfix, replacing original file\n{dcm2niixfile} ->\n{newbidsfile}")
+ dcm2niixfile.replace(newbidsfile)
+
+ # Check if there are files that got additional postfixes from dcm2niix. See: https://github.com/rordenlab/dcm2niix/blob/master/FILENAMING.md
+ dcm2niixpostfixes = ('_c', '_i', '_Eq', '_real', '_imaginary', '_MoCo', '_t', '_Tilt', '_e', '_ph', '_ADC', '_fieldmaphz') #_c%d, _e%d and _ph (and any combination of these in that order) are for multi-coil data, multi-echo data and phase data
+ dcm2niixfiles = sorted(set(dcm2niixfile for dcm2niixpostfix in dcm2niixpostfixes for dcm2niixfile in outfolder.glob(f"{bidsname}*{dcm2niixpostfix}*.nii*")))
+ dcm2niixfiles = [dcm2niixfile for dcm2niixfile in dcm2niixfiles if not (re.match(r'sub-.*_echo-[0-9]*\.nii', dcm2niixfile.name) or
+ re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))] # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see GitHub issue #232)
+
+ # Rename all dcm2niix files that got additional postfixes (i.e. store the postfixes in the bidsname)
+ for dcm2niixfile in dcm2niixfiles:
+
+ # Strip each dcm2niix postfix and assign it to the proper bids entity in the new bidsname (else assign it to the fallback entity)
+ ext = ''.join(dcm2niixfile.suffixes)
+ postfixes = dcm2niixfile.name.split(bidsname)[1].rsplit(ext)[0].split('_')[1:]
+ newbidsname = bids.insert_bidskeyval(dcm2niixfile.name, 'run', runindex, ignore) # Restart the run-index. NB: Unlike bidsname, newbidsname has a file extension
+ for postfix in postfixes:
+
+ # Patch the echo entity in the newbidsname with the dcm2niix echo info # NB: We can't rely on the bids-entity info here because manufacturers can e.g. put multiple echos in one series/run-folder
+ if 'echo' in run.bids and postfix.startswith('e'):
+ echonr = f"_{postfix}".replace('_e','') # E.g. postfix='e1'
+ if not echonr:
+ echonr = '1'
+ if echonr.isdecimal():
+ newbidsname = bids.insert_bidskeyval(newbidsname, 'echo', echonr.lstrip('0'), ignore) # In contrast to other labels, run and echo labels MUST be integers. Those labels MAY include zero padding, but this is NOT RECOMMENDED to maintain their uniqueness
+ elif echonr[0:-1].isdecimal():
+ LOGGER.verbose(f"Splitting off echo-number {echonr[0:-1]} from the '{postfix}' postfix")
+ newbidsname = bids.insert_bidskeyval(newbidsname, 'echo', echonr[0:-1].lstrip('0'), ignore) # Strip of the 'a', 'b', etc. from `e1a`, `e1b`, etc
+ newbidsname = bids.get_bidsvalue(newbidsname, fallback, echonr[-1]) # Append the 'a' to the fallback-label
+ else:
+ LOGGER.error(f"Unexpected postix '{postfix}' found in {dcm2niixfile}")
+ newbidsname = bids.get_bidsvalue(newbidsname, fallback, postfix) # Append the unknown postfix to the fallback-label
+
+ # Patch the phase entity in the newbidsname with the dcm2niix mag/phase info
+ elif 'part' in run.bids and postfix in ('ph','real','imaginary'): # e.g. part: ['', 'mag', 'phase', 'real', 'imag', 0]
+ if postfix == 'ph':
+ newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'phase', ignore)
+ if postfix == 'real':
+ newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'real', ignore)
+ if postfix == 'imaginary':
+ newbidsname = bids.insert_bidskeyval(newbidsname, 'part', 'imag', ignore)
+
+ # Patch fieldmap images (NB: datatype=='fmap' is too broad, see the fmap.yaml file)
+ elif suffix in bids.filerules.fmap.fieldmaps.suffixes: # i.e. in ('magnitude','magnitude1','magnitude2','phase1','phase2','phasediff','fieldmap')
+ if len(dcm2niixfiles) not in (1, 2, 3, 4): # Phase/echo data may be stored in the same data source/run folder
+ LOGGER.verbose(f"Unknown fieldmap {outfolder/bidsname} for '{postfix}'")
+ newbidsname = newbidsname.replace('_magnitude1a', '_magnitude2') # First catch this potential weird/rare case
+ newbidsname = newbidsname.replace('_magnitude1_pha', '_phase2') # First catch this potential weird/rare case
+ newbidsname = newbidsname.replace('_magnitude1_e1', '_magnitude1') # Case 2 = Two phase and magnitude images
+ newbidsname = newbidsname.replace('_magnitude1_e2', '_magnitude2') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
+ newbidsname = newbidsname.replace('_magnitude2_e1', '_magnitude1') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
+ newbidsname = newbidsname.replace('_magnitude2_e2', '_magnitude2') # Case 2
+ if len(dcm2niixfiles) in (2,3): # Case 1 = One or two magnitude + one phasediff image
+ newbidsname = newbidsname.replace('_magnitude1_ph', '_phasediff')
+ newbidsname = newbidsname.replace('_magnitude2_ph', '_phasediff')
+ newbidsname = newbidsname.replace('_phasediff_e1', '_phasediff') # Case 1
+ newbidsname = newbidsname.replace('_phasediff_e2', '_phasediff') # Case 1
+ newbidsname = newbidsname.replace('_phasediff_ph', '_phasediff') # Case 1
+ newbidsname = newbidsname.replace('_magnitude1_ph', '_phase1') # Case 2: One or two magnitude and phase images in one folder
+ newbidsname = newbidsname.replace('_magnitude2_ph', '_phase2') # Case 2: Two magnitude + two phase images in one folder
+ newbidsname = newbidsname.replace('_phase1_e1', '_phase1') # Case 2
+ newbidsname = newbidsname.replace('_phase1_e2', '_phase2') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
+ newbidsname = newbidsname.replace('_phase2_e1', '_phase1') # Case 2: This can happen when the e2 image is stored in the same directory as the e1 image, but with the e2 listed first
+ newbidsname = newbidsname.replace('_phase2_e2', '_phase2') # Case 2
+ newbidsname = newbidsname.replace('_phase1_ph', '_phase1') # Case 2: One or two magnitude and phase images in one folder
+ newbidsname = newbidsname.replace('_phase2_ph', '_phase2') # Case 2: Two magnitude + two phase images in one folder
+ newbidsname = newbidsname.replace('_magnitude_e1', '_magnitude') # Case 3 = One magnitude + one fieldmap image
+ if len(dcm2niixfiles) == 2:
+ newbidsname = newbidsname.replace('_fieldmap_e1', '_magnitude') # Case 3: One magnitude + one fieldmap image in one folder
+ newbidsname = newbidsname.replace('_magnitude_fieldmaphz', '_fieldmap')
+ newbidsname = newbidsname.replace('_fieldmap_fieldmaphz', '_fieldmap')
+ newbidsname = newbidsname.replace('_fieldmap_e1', '_fieldmap') # Case 3
+ newbidsname = newbidsname.replace('_magnitude_ph', '_fieldmap') # Case 3: One magnitude + one fieldmap image in one folder
+ newbidsname = newbidsname.replace('_fieldmap_ph', '_fieldmap') # Case 3
+
+ # Append the dcm2niix info to the fallback-label, may need to be improved/elaborated for future BIDS standards, supporting multi-coil data
+ else:
+ newbidsname = bids.get_bidsvalue(newbidsname, fallback, postfix)
+
+ # Remove the added postfix from the new bidsname
+ newbidsname = newbidsname.replace(f"_{postfix}_",'_') # If it is not last
+ newbidsname = newbidsname.replace(f"_{postfix}.",'.') # If it is last
+
+ # The ADC images are not BIDS compliant
+ if postfix == 'ADC':
+ LOGGER.warning(f"The {newbidsname} image is a derivate / not BIDS-compliant -- you can probably delete it safely and update {scans_tsv}")
+
+ # Save the NIfTI file with the newly constructed name
+ newbidsname = run.increment_runindex(outfolder, newbidsname, scans_table, targets) # Update the runindex now that the name has changed
+ newbidsfile = outfolder/newbidsname
+ LOGGER.verbose(f"Found dcm2niix {postfixes} postfixes, renaming\n{dcm2niixfile} ->\n{newbidsfile}")
+ if newbidsfile.is_file():
+ LOGGER.warning(f"Overwriting existing {newbidsfile} file -- check your results carefully!")
+ dcm2niixfile.replace(newbidsfile)
+ targets.add(newbidsfile)
+
+ # Rename all associated files (i.e. the json-, bval- and bvec-files)
+ for oldfile in outfolder.glob(dcm2niixfile.with_suffix('').stem + '.*'):
+ oldfile.replace(newbidsfile.with_suffix('').with_suffix(''.join(oldfile.suffixes)))
+
+ # Write out provenance data
+ bids.bidsprov(bidsses, source, run, targets)
+
+ # Loop over all targets (i.e. the produced output files) and edit the json sidecar data
+ for target in sorted(targets):
+
+ # Load/copy over the source meta-data
+ jsonfile = target.with_suffix('').with_suffix('.json')
+ if not jsonfile.is_file():
+ LOGGER.warning(f"Unexpected conversion result, could not find: {jsonfile}")
+ metadata = bids.poolmetadata(run.datasource, jsonfile, run.meta, options.get('meta',[]))
+
+ # Remove the bval/bvec files of sbref- and inv-images (produced by dcm2niix but not allowed by the BIDS specifications)
+ if ((run.datatype == 'dwi' and suffix == 'sbref') or
+ (run.datatype == 'fmap' and suffix == 'epi') or
+ (run.datatype == 'anat' and suffix == 'MP2RAGE')):
+ for ext in ('.bval', '.bvec'):
+ bfile = target.with_suffix('').with_suffix(ext)
+ if bfile.is_file() and not bids.check_ignore(bfile.name, bidsignore, 'file'):
+ bdata = pd.read_csv(bfile, header=None)
+ if bdata.any(axis=None):
+ LOGGER.warning(f"Found unexpected non-zero b-values in '{bfile}'")
+ LOGGER.verbose(f"Removing BIDS-invalid b0-file: {bfile} -> {jsonfile}")
+ metadata[ext[1:]] = bdata.values.tolist()
+ bfile.unlink()
+
+ # Save the meta-data to the json sidecar-file
+ if metadata:
+ with jsonfile.open('w') as json_fid:
+ json.dump(metadata, json_fid, indent=4)
+
+ # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
+ if not ignore:
+ acq_time = ''
+ if dataformat == 'DICOM':
+ acq_time = f"{run.datasource.attributes('AcquisitionDate')}T{run.datasource.attributes('AcquisitionTime')}"
+ elif dataformat == 'PAR':
+ acq_time = run.datasource.attributes('exam_date')
+ if not acq_time or acq_time == 'T':
+ acq_time = f"1925-01-01T{metadata.get('AcquisitionTime','')}"
+ try:
+ acq_time = dateutil.parser.parse(acq_time)
+ if options.get('anon','y') in ('y','yes'):
+ acq_time = acq_time.replace(year=1925, month=1, day=1) # Privacy protection (see BIDS specification)
+ acq_time = acq_time.isoformat()
+ except Exception as jsonerror:
+ LOGGER.warning(f"Could not parse the acquisition time from: {sourcefile}\n{jsonerror}")
+ acq_time = 'n/a'
+ scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time
+
+ # Check if the target output aligns with dcm2niix's "BidsGuess" datatype and filename entities
+ if not ignore:
+ typeguess, targetguess = metadata.get('BidsGuess') or ['', ''] # BidsGuess: [datatype, filename]
+ LOGGER.bcdebug(f"BidsGuess: [{typeguess}, {targetguess}]")
+ if typeguess and run.datatype != typeguess:
+ LOGGER.info(f"The datatype of {target.relative_to(bidsses)} does not match with the datatype guessed by dcm2niix: {typeguess}")
+ elif targetguess and bids.get_bidsvalue(target, 'suffix') != bids.get_bidsvalue(targetguess, 'suffix'):
+ LOGGER.info(f"The suffix of {target.relative_to(bidsses)} does not match with the suffix guessed by dcm2niix: {targetguess}")
+ for entity in ('part', 'inv', 'echo', 'dir'):
+ targetvalue = bids.get_bidsvalue(target, entity)
+ guessvalue = bids.get_bidsvalue(targetguess, entity)
+ if targetvalue and guessvalue and targetvalue != guessvalue:
+ LOGGER.warning(f"The '{entity}_{targetvalue}' value in {target.relative_to(bidsses)} does not match with the '{entity}_{guessvalue}' value guessed by dcm2niix: {targetguess}")
+
+ # Write the scans_table to disk
+ LOGGER.verbose(f"Writing acquisition time data to: {scans_tsv}")
+ scans_table.sort_values(by=['acq_time','filename'], inplace=True)
+ scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
+
+ # Collect personal data for the participants.tsv file
+ if dataformat == 'DICOM': # PAR does not contain personal info?
+ age = datasource.attributes('PatientAge') # A string of characters with one of the following formats: nnnD, nnnW, nnnM, nnnY
+ try:
+ if age.endswith('D'): age = float(age.rstrip('D')) / 365.2524
+ elif age.endswith('W'): age = float(age.rstrip('W')) / 52.1775
+ elif age.endswith('M'): age = float(age.rstrip('M')) / 12
+ elif age.endswith('Y'): age = float(age.rstrip('Y'))
+ if age and options.get('anon','y') in ('y','yes'):
+ age = int(float(age))
+ except Exception as exc:
+ LOGGER.warning(f"Could not parse age from: {datasource}\n{exc}")
+ personals = {}
+ personals['age'] = str(age)
+ personals['sex'] = datasource.attributes('PatientSex')
+ personals['size'] = datasource.attributes('PatientSize')
+ personals['weight'] = datasource.attributes('PatientWeight')
+
+ return personals
diff --git a/bidscoin/plugins/events2bids.py b/bidscoin/plugins/events2bids.py
index 36f33a19..b1dc2e87 100644
--- a/bidscoin/plugins/events2bids.py
+++ b/bidscoin/plugins/events2bids.py
@@ -7,9 +7,9 @@
from bids_validator import BIDSValidator
from pathlib import Path
from bidscoin import bids
-from bidscoin.plugins import EventsParser
+from bidscoin.plugins import PluginInterface, EventsParser
from bidscoin.bids import BidsMap, DataFormat, is_hidden, Plugin
-# from convert_eprime.utils import remove_unicode
+# TODO: from convert_eprime.utils import ..
LOGGER = logging.getLogger(__name__)
@@ -17,213 +17,148 @@
OPTIONS = Plugin({'table': 'event', 'skiprows': 3, 'meta': ['.json', '.tsv']}) # The file extensions of the equally named metadata sourcefiles that are copied over as BIDS sidecar files
-def test(options: Plugin=OPTIONS) -> int:
- """
- Performs a Presentation test
+class Interface(PluginInterface):
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['events2bids']
- :return: The errorcode: 0 for successful execution, 1 for general tool errors, 2 for `ext` option errors, 3 for `meta` option errors
- """
-
- LOGGER.info('Testing the events2bids installation:')
-
- # Test the Presentation installation
- try:
- pass
-
- except Exception as eventserror:
-
- LOGGER.error(f"Events2bids error:\n{eventserror}")
- return 1
-
- return 0
+ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The valid/supported dataformat of the sourcefile
+ """
-def has_support(file: Path, dataformat: Union[DataFormat, str]='') -> str:
- """
- This plugin function assesses whether a sourcefile is of a supported dataformat
+ if dataformat and dataformat != 'Presentation':
+ return ''
- :param file: The sourcefile that is assessed
- :param dataformat: The requested dataformat (optional requirement)
- :return: The valid/supported dataformat of the sourcefile
- """
+ ext = ''.join(file.suffixes)
+ if ext.lower() in ('.log',):
+ return 'Presentation'
- if dataformat and dataformat != 'Presentation':
return ''
- ext = ''.join(file.suffixes)
- if ext.lower() in ('.log',):
- return 'Presentation'
-
- return ''
-
-
-def get_attribute(dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options: Plugin) -> Union[str, int, float, list]:
- """
- This plugin supports reading attributes from DICOM and PAR dataformats
-
- :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
- :param sourcefile: The sourcefile from which the attribute value should be read
- :param attribute: The attribute key for which the value should be read
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['events2bids']
- :return: The retrieved attribute value
- """
-
- if dataformat == 'Presentation':
-
- try:
- with sourcefile.open() as fid:
- while '-' in (line := fid.readline()):
- key, value = line.split('-', 1)
- if attribute == key.strip():
- return value.strip()
-
- except (IOError, OSError) as ioerror:
- LOGGER.exception(f"Could not get the Presentation '{attribute}' attribute from {sourcefile}\n{ioerror}")
-
- return ''
-
-
-def bidsmapper_plugin(session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
- """
- The goal of this plugin function is to identify all the different runs in the session and update the
- bidsmap if a new run is discovered
+ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options) -> Union[str, int, float, list]:
+ """
+ This plugin supports reading attributes from DICOM and PAR dataformats
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap_new: The new study bidsmap that we are building
- :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
- :param template: The template bidsmap with the default heuristics
- """
+ :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which the attribute value should be read
+ :param attribute: The attribute key for which the value should be read
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['events2bids']
+ :return: The retrieved attribute value
+ """
- # See for every source file in the session if we already discovered it or not
- for sourcefile in session.rglob('*'):
+ if dataformat == 'Presentation':
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- if not is_hidden(sourcefile.relative_to(session)):
- LOGGER.bcdebug(f"Skipping {sourcefile} (not supported by {dataformat})")
- continue
+ try:
+ with sourcefile.open() as fid:
+ while '-' in (line := fid.readline()):
+ key, value = line.split('-', 1)
+ if attribute == key.strip():
+ return value.strip()
- # See if we can find a matching run in the old bidsmap
- run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+ except (IOError, OSError) as ioerror:
+ LOGGER.exception(f"Could not get the Presentation '{attribute}' attribute from {sourcefile}\n{ioerror}")
- # If not, see if we can find a matching run in the template
- if not oldmatch:
- run, _ = template.get_matching_run(sourcefile, dataformat)
+ return ''
- # See if we have already put the run somewhere in our new bidsmap
- if not bidsmap_new.exist_run(run):
+ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> None:
+ """
+ The bidscoiner plugin to convert the session Presentation source-files into BIDS-valid NIfTI-files in the
+ corresponding bids session-folder
- # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
- if not oldmatch:
- LOGGER.info(f"Discovered sample: {run.datasource}")
- else:
- LOGGER.bcdebug(f"Known sample: {run.datasource}")
+ :param session: The full-path name of the subject/session source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: Nothing (i.e. personal data is not available)
+ """
- # Copy the filled-in run over to the new bidsmap
- bidsmap_new.insert_run(run)
+ # Get the subject identifiers from the bidsses folder
+ subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
+ sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
+ options = bidsmap.plugins['events2bids']
+ runid = ''
+ # Read or create a scans_table and tsv-file
+ scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
+ if scans_tsv.is_file():
+ scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
else:
- LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
-
-
-def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> None:
- """
- The bidscoiner plugin to convert the session Presentation source-files into BIDS-valid NIfTI-files in the
- corresponding bids session-folder
-
- :param session: The full-path name of the subject/session source folder
- :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
- :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
- :return: Nothing (i.e. personal data is not available)
- """
-
- # Get the subject identifiers from the bidsses folder
- subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
- sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
- options = bidsmap.plugins['events2bids']
- runid = ''
-
- # Read or create a scans_table and tsv-file
- scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
- if scans_tsv.is_file():
- scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
- else:
- scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
- scans_table.index.name = 'filename'
-
- # Collect the different Presentation source files for all files in the session
- for sourcefile in session.rglob('*'):
-
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
-
- # Get a matching run from the bidsmap
- run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
-
- # Check if we should ignore this run
- if run.datatype in bidsmap.options['ignoretypes']:
- LOGGER.info(f"--> Leaving out: {run.datasource}")
- bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
- continue
-
- # Check if we already know this run
+ scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
+ scans_table.index.name = 'filename'
+
+ # Collect the different Presentation source files for all files in the session
+ for sourcefile in session.rglob('*'):
+
+ # Check if the sourcefile is of a supported dataformat
+ if is_hidden(sourcefile.relative_to(session)) or not (dataformat := self.has_support(sourcefile)):
+ continue
+
+ # Get a matching run from the bidsmap
+ run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
+
+ # Check if we should ignore this run
+ if run.datatype in bidsmap.options['ignoretypes']:
+ LOGGER.info(f"--> Leaving out: {run.datasource}")
+ bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
+ continue
+
+ # Check if we already know this run
+ if not runid:
+ LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
+ bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
+ continue
+
+ LOGGER.info(f"--> Coining: {run.datasource}")
+
+ # Create the BIDS session/datatype output folder
+ outfolder = bidsses/run.datatype
+ outfolder.mkdir(parents=True, exist_ok=True)
+
+ # Compose the BIDS filename using the matched run
+ bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
+ bidsname = run.bidsname(subid, sesid, not bidsignore, runtime=True)
+ bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
+ bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
+ eventsfile = (outfolder/bidsname).with_suffix('.tsv')
+
+ # Check if the bidsname is valid
+ bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
+ isbids = BIDSValidator().is_bids(bidstest)
+ if not isbids and not bidsignore:
+ LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
+
+ # Check if file already exists (-> e.g. when a static runindex is used)
+ if eventsfile.is_file():
+ LOGGER.warning(f"{eventsfile} already exists and will be deleted -- check your results carefully!")
+ eventsfile.unlink()
+
+ # Save the sourcefile as a BIDS NIfTI file and write out provenance logging data
+ run.eventsparser().write(eventsfile)
+ bids.bidsprov(bidsses, sourcefile, run, [eventsfile] if eventsfile.is_file() else [])
+
+ # Check the output
+ if not eventsfile.is_file():
+ LOGGER.error(f"Output file not found: {eventsfile}")
+ continue
+
+ # Load/copy over the source meta-data
+ sidecar = eventsfile.with_suffix('.json')
+ metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta', []))
+ if metadata:
+ with sidecar.open('w') as json_fid:
+ json.dump(metadata, json_fid, indent=4)
+
+ # Add an entry to the scans_table (we typically don't have useful data to put there)
+ scans_table.loc[eventsfile.relative_to(bidsses).as_posix(), 'acq_time'] = 'n/a'
+
if not runid:
- LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
- bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
- continue
-
- LOGGER.info(f"--> Coining: {run.datasource}")
-
- # Create the BIDS session/datatype output folder
- outfolder = bidsses/run.datatype
- outfolder.mkdir(parents=True, exist_ok=True)
-
- # Compose the BIDS filename using the matched run
- bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
- bidsname = run.bidsname(subid, sesid, not bidsignore, runtime=True)
- bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
- bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
- eventsfile = (outfolder/bidsname).with_suffix('.tsv')
-
- # Check if the bidsname is valid
- bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
- isbids = BIDSValidator().is_bids(bidstest)
- if not isbids and not bidsignore:
- LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
-
- # Check if file already exists (-> e.g. when a static runindex is used)
- if eventsfile.is_file():
- LOGGER.warning(f"{eventsfile} already exists and will be deleted -- check your results carefully!")
- eventsfile.unlink()
-
- # Save the sourcefile as a BIDS NIfTI file and write out provenance logging data
- run.eventsparser().write(eventsfile)
- bids.bidsprov(bidsses, sourcefile, run, [eventsfile] if eventsfile.is_file() else [])
-
- # Check the output
- if not eventsfile.is_file():
- LOGGER.error(f"Output file not found: {eventsfile}")
- continue
-
- # Load/copy over the source meta-data
- sidecar = eventsfile.with_suffix('.json')
- metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta', []))
- if metadata:
- with sidecar.open('w') as json_fid:
- json.dump(metadata, json_fid, indent=4)
-
- # Add an entry to the scans_table (we typically don't have useful data to put there)
- scans_table.loc[eventsfile.relative_to(bidsses).as_posix(), 'acq_time'] = 'n/a'
-
- if not runid:
- LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
- return
-
- # Write the scans_table to disk
- LOGGER.verbose(f"Writing data to: {scans_tsv}")
- scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
+ LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
+ return
+
+ # Write the scans_table to disk
+ LOGGER.verbose(f"Writing data to: {scans_tsv}")
+ scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
class PresentationEvents(EventsParser):
diff --git a/bidscoin/plugins/nibabel2bids.py b/bidscoin/plugins/nibabel2bids.py
index af5cb547..73a944f4 100644
--- a/bidscoin/plugins/nibabel2bids.py
+++ b/bidscoin/plugins/nibabel2bids.py
@@ -12,6 +12,7 @@
from pathlib import Path
from bidscoin import bids
from bidscoin.bids import BidsMap, DataFormat, Plugin, is_hidden
+from bidscoin.plugins import PluginInterface
try:
from nibabel.testing import data_path
@@ -26,221 +27,182 @@
'meta': ['.json', '.tsv', '.tsv.gz', '.bval', '.bvec']}) # The file extensions of the equally named metadata sourcefiles that are copied over as BIDS sidecar files
-def test(options: Plugin=OPTIONS) -> int:
- """
- Performs a nibabel test
+class Interface(PluginInterface):
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['nibabel2bids']
- :return: The errorcode: 0 for successful execution, 1 for general tool errors, 2 for `ext` option errors, 3 for `meta` option errors
- """
+ def test(self, options: Plugin=OPTIONS) -> int:
+ """
+ Performs a nibabel test
- LOGGER.info('Testing the nibabel2bids installation:')
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['nibabel2bids']
+ :return: The errorcode: 0 for successful execution, 1 for general tool errors, 2 for `ext` option errors, 3 for `meta` option errors
+ """
- # Test the nibabel installation
- try:
+ LOGGER.info('Testing the nibabel2bids installation:')
- LOGGER.info(f"Nibabel version: {nib.__version__}")
- if options.get('ext',OPTIONS['ext']) not in ('.nii', '.nii.gz'):
- LOGGER.error(f"The 'ext: {options.get('ext')}' value in the nibabel2bids options is not '.nii' or '.nii.gz'")
- return 2
-
- if not isinstance(options.get('meta',OPTIONS['meta']), list):
- LOGGER.error(f"The 'meta: {options.get('meta')}' value in the nibabel2bids options is not a list")
- return 3
-
- niifile = Path(data_path)/'anatomical.nii'
- assert has_support(niifile) == 'Nibabel'
- assert str(get_attribute('Nibabel', niifile, 'descrip', options)) == "b'spm - 3D normalized'"
-
- except Exception as nibabelerror:
-
- LOGGER.error(f"Nibabel error:\n{nibabelerror}")
- return 1
+ # Test the nibabel installation
+ try:
- return 0
+ LOGGER.info(f"Nibabel version: {nib.__version__}")
+ if options.get('ext',OPTIONS['ext']) not in ('.nii', '.nii.gz'):
+ LOGGER.error(f"The 'ext: {options.get('ext')}' value in the nibabel2bids options is not '.nii' or '.nii.gz'")
+ return 2
+ if not isinstance(options.get('meta',OPTIONS['meta']), list):
+ LOGGER.error(f"The 'meta: {options.get('meta')}' value in the nibabel2bids options is not a list")
+ return 3
-def has_support(file: Path, dataformat: Union[DataFormat, str]='') -> str:
- """
- This plugin function assesses whether a sourcefile is of a supported dataformat
+ niifile = Path(data_path)/'anatomical.nii'
+ assert self.has_support(niifile) == 'Nibabel'
+ assert str(self.get_attribute('Nibabel', niifile, 'descrip', options)) == "b'spm - 3D normalized'"
- :param file: The sourcefile that is assessed
- :param dataformat: The requested dataformat (optional requirement)
- :return: The valid/supported dataformat of the sourcefile
- """
+ except Exception as nibabelerror:
- if dataformat and dataformat != 'Nibabel':
- return ''
+ LOGGER.error(f"Nibabel error:\n{nibabelerror}")
+ return 1
- ext = ''.join(file.suffixes)
- if file.is_file() and ext.lower() in sum((klass.valid_exts for klass in nib.imageclasses.all_image_classes), ('.nii.gz',)):
- return 'Nibabel'
+ return 0
- return ''
+ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
-def get_attribute(dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options: Plugin) -> Union[str, int, float, list]:
- """
- This plugin supports reading attributes from DICOM and PAR dataformats
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The valid/supported dataformat of the sourcefile
+ """
- :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
- :param sourcefile: The sourcefile from which the attribute value should be read
- :param attribute: The attribute key for which the value should be read
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['nibabel2bids']
- :return: The retrieved attribute value
- """
+ if dataformat and dataformat != 'Nibabel':
+ return ''
- value = None
+ ext = ''.join(file.suffixes)
+ if file.is_file() and ext.lower() in sum((klass.valid_exts for klass in nib.imageclasses.all_image_classes), ('.nii.gz',)):
+ return 'Nibabel'
- if dataformat == 'Nibabel':
-
- try:
- value = nib.load(sourcefile).header.get(attribute)
- if value is not None:
- value = value.tolist()
+ return ''
- except Exception:
- LOGGER.exception(f"Could not get the nibabel '{attribute}' attribute from {sourcefile} -> {value}")
- return value
+ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options: Plugin) -> Union[str, int, float, list]:
+ """
+ This plugin supports reading attributes from DICOM and PAR dataformats
+ :param dataformat: The bidsmap-dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which the attribute value should be read
+ :param attribute: The attribute key for which the value should be read
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['nibabel2bids']
+ :return: The retrieved attribute value
+ """
-def bidsmapper_plugin(session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
- """
- The goal of this plugin function is to identify all the different runs in the session and update the
- bidsmap if a new run is discovered
+ value = None
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap_new: The new study bidsmap that we are building
- :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
- :param template: The template bidsmap with the default heuristics
- """
+ if dataformat == 'Nibabel':
- # See for every source file in the session if we already discovered it or not
- for sourcefile in session.rglob('*'):
+ try:
+ value = nib.load(sourcefile).header.get(attribute)
+ if value is not None:
+ value = value.tolist()
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
+ except Exception:
+ LOGGER.exception(f"Could not get the nibabel '{attribute}' attribute from {sourcefile} -> {value}")
- # See if we can find a matching run in the old bidsmap
- run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+ return value
- # If not, see if we can find a matching run in the template
- if not oldmatch:
- run, _ = template.get_matching_run(sourcefile, dataformat)
- # See if we have already put the run somewhere in our new bidsmap
- if not bidsmap_new.exist_run(run):
+ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> None:
+ """
+ The bidscoiner plugin to convert the session Nibabel source-files into BIDS-valid NIfTI-files in the
+ corresponding bids session-folder
- # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
- if not oldmatch:
- LOGGER.info(f"Discovered sample: {run.datasource}")
- else:
- LOGGER.bcdebug(f"Known sample: {run.datasource}")
+ :param session: The full-path name of the subject/session source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: Nothing (i.e. personal data is not available)
+ """
- # Copy the filled-in run over to the new bidsmap
- bidsmap_new.insert_run(run)
+ # Get the subject identifiers from the bidsses folder
+ subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
+ sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
+ options = bidsmap.plugins['nibabel2bids']
+ runid = ''
+ # Read or create a scans_table and tsv-file
+ scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
+ if scans_tsv.is_file():
+ scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
else:
- LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
-
-
-def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> None:
- """
- The bidscoiner plugin to convert the session Nibabel source-files into BIDS-valid NIfTI-files in the
- corresponding bids session-folder
-
- :param session: The full-path name of the subject/session source folder
- :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
- :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
- :return: Nothing (i.e. personal data is not available)
- """
-
- # Get the subject identifiers from the bidsses folder
- subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
- sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
- options = bidsmap.plugins['nibabel2bids']
- runid = ''
-
- # Read or create a scans_table and tsv-file
- scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
- if scans_tsv.is_file():
- scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
- else:
- scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
- scans_table.index.name = 'filename'
-
- # Collect the different Nibabel source files for all files in the session
- for sourcefile in session.rglob('*'):
-
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
-
- # Get a matching run from the bidsmap
- run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
-
- # Check if we should ignore this run
- if run.datatype in bidsmap.options['ignoretypes']:
- LOGGER.info(f"--> Leaving out: {run.datasource}")
- bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
- continue
-
- # Check if we already know this run
+ scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
+ scans_table.index.name = 'filename'
+
+ # Collect the different Nibabel source files for all files in the session
+ for sourcefile in session.rglob('*'):
+
+ # Check if the sourcefile is of a supported dataformat
+ if is_hidden(sourcefile.relative_to(session)) or not (dataformat := self.has_support(sourcefile)):
+ continue
+
+ # Get a matching run from the bidsmap
+ run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
+
+ # Check if we should ignore this run
+ if run.datatype in bidsmap.options['ignoretypes']:
+ LOGGER.info(f"--> Leaving out: {run.datasource}")
+ bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
+ continue
+
+ # Check if we already know this run
+ if not runid:
+ LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
+ bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
+ continue
+
+ LOGGER.info(f"--> Coining: {run.datasource}")
+
+ # Create the BIDS session/datatype output folder
+ outfolder = bidsses/run.datatype
+ outfolder.mkdir(parents=True, exist_ok=True)
+
+ # Compose the BIDS filename using the matched run
+ bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
+ bidsname = run.bidsname(subid, sesid, not bidsignore, runtime=True)
+ bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
+ bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
+ target = (outfolder/bidsname).with_suffix(options.get('ext', ''))
+
+ # Check if the bidsname is valid
+ bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
+ isbids = BIDSValidator().is_bids(bidstest)
+ if not isbids and not bidsignore:
+ LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
+
+ # Check if file already exists (-> e.g. when a static runindex is used)
+ if target.is_file():
+ LOGGER.warning(f"{target} already exists and will be deleted -- check your results carefully!")
+ target.unlink()
+
+ # Save the sourcefile as a BIDS NIfTI file and write out provenance logging data
+ nib.save(nib.load(sourcefile), target)
+ bids.bidsprov(bidsses, sourcefile, run, [target] if target.is_file() else [])
+
+ # Check the output
+ if not target.is_file():
+ LOGGER.error(f"Output file not found: {target}")
+ continue
+
+ # Load/copy over the source meta-data
+ sidecar = target.with_suffix('').with_suffix('.json')
+ metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta', []))
+ if metadata:
+ with sidecar.open('w') as json_fid:
+ json.dump(metadata, json_fid, indent=4)
+
+ # Add an entry to the scans_table (we typically don't have useful data to put there)
+ acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}")
+ scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat()
+
if not runid:
- LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete {bidsses} to solve this warning")
- bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
- continue
-
- LOGGER.info(f"--> Coining: {run.datasource}")
-
- # Create the BIDS session/datatype output folder
- outfolder = bidsses/run.datatype
- outfolder.mkdir(parents=True, exist_ok=True)
-
- # Compose the BIDS filename using the matched run
- bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
- bidsname = run.bidsname(subid, sesid, not bidsignore, runtime=True)
- bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
- bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
- target = (outfolder/bidsname).with_suffix(options.get('ext', ''))
-
- # Check if the bidsname is valid
- bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
- isbids = BIDSValidator().is_bids(bidstest)
- if not isbids and not bidsignore:
- LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
-
- # Check if file already exists (-> e.g. when a static runindex is used)
- if target.is_file():
- LOGGER.warning(f"{target} already exists and will be deleted -- check your results carefully!")
- target.unlink()
-
- # Save the sourcefile as a BIDS NIfTI file and write out provenance logging data
- nib.save(nib.load(sourcefile), target)
- bids.bidsprov(bidsses, sourcefile, run, [target] if target.is_file() else [])
-
- # Check the output
- if not target.is_file():
- LOGGER.error(f"Output file not found: {target}")
- continue
-
- # Load/copy over the source meta-data
- sidecar = target.with_suffix('').with_suffix('.json')
- metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta', []))
- if metadata:
- with sidecar.open('w') as json_fid:
- json.dump(metadata, json_fid, indent=4)
-
- # Add an entry to the scans_table (we typically don't have useful data to put there)
- acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}")
- scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat()
-
- if not runid:
- LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
- return
-
- # Write the scans_table to disk
- LOGGER.verbose(f"Writing data to: {scans_tsv}")
- scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
+ LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
+ return
+
+ # Write the scans_table to disk
+ LOGGER.verbose(f"Writing data to: {scans_tsv}")
+ scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
diff --git a/bidscoin/plugins/spec2nii2bids.py b/bidscoin/plugins/spec2nii2bids.py
index e41787fe..8c411e7f 100644
--- a/bidscoin/plugins/spec2nii2bids.py
+++ b/bidscoin/plugins/spec2nii2bids.py
@@ -12,6 +12,7 @@
from pathlib import Path
from bidscoin import run_command, bids, due, Doi
from bidscoin.bids import BidsMap, DataFormat, Plugin, is_hidden
+from bidscoin.plugins import PluginInterface
LOGGER = logging.getLogger(__name__)
@@ -23,289 +24,249 @@
'multiraid': 2}) # The mapVBVD argument for selecting the multiraid Twix file to load (default = 2, i.e. 2nd file)
-def test(options: Plugin=OPTIONS) -> int:
- """
- This plugin shell tests the working of the spec2nii2bids plugin + given options
+class Interface(PluginInterface):
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['spec2nii2bids']
- :return: The errorcode (e.g 0 if the tool generated the expected result, > 0 if there was a tool error)
- """
+ def test(self, options: Plugin=OPTIONS) -> int:
+ """
+ This plugin shell tests the working of the spec2nii2bids plugin + given options
- LOGGER.info('Testing the spec2nii2bids installation:')
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins['spec2nii2bids']
+ :return: The errorcode (e.g 0 if the tool generated the expected result, > 0 if there was a tool error)
+ """
- if not all(hasattr(bids, name) for name in ('get_twixfield', 'get_sparfield', 'get_p7field')):
- LOGGER.error("Could not import the expected 'get_twixfield', 'get_sparfield' and/or 'get_p7field' from the bids.py library")
- return 1
- if 'command' not in {**OPTIONS, **options}:
- LOGGER.error(f"The expected 'command' key is not defined in the spec2nii2bids options")
- return 1
- if 'args' not in {**OPTIONS, **options}:
- LOGGER.warning(f"The expected 'args' key is not defined in the spec2nii2bids options")
+ LOGGER.info('Testing the spec2nii2bids installation:')
- # Test the spec2nii installation
- return run_command(f"{options.get('command',OPTIONS['command'])} -v")
+ if not all(hasattr(bids, name) for name in ('get_twixfield', 'get_sparfield', 'get_p7field')):
+ LOGGER.error("Could not import the expected 'get_twixfield', 'get_sparfield' and/or 'get_p7field' from the bids.py library")
+ return 1
+ if 'command' not in {**OPTIONS, **options}:
+ LOGGER.error(f"The expected 'command' key is not defined in the spec2nii2bids options")
+ return 1
+ if 'args' not in {**OPTIONS, **options}:
+ LOGGER.warning(f"The expected 'args' key is not defined in the spec2nii2bids options")
+ # Test the spec2nii installation
+ return run_command(f"{options.get('command',OPTIONS['command'])} -v")
-def has_support(file: Path, dataformat: Union[DataFormat, str]='') -> str:
- """
- This plugin function assesses whether a sourcefile is of a supported dataformat
- :param file: The sourcefile that is assessed
- :param dataformat: The requested dataformat (optional requirement)
- :return: The valid/supported dataformat of the sourcefile
- """
+ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
- if dataformat and dataformat not in ('Twix', 'SPAR', 'Pfile'):
- return ''
-
- suffix = file.suffix.lower()
- if suffix == '.dat':
- return 'Twix'
- elif suffix == '.spar':
- return 'SPAR'
- elif suffix == '.7' and not bids.is_dicomfile(file):
- return 'Pfile'
-
- return ''
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The valid/supported dataformat of the sourcefile
+ """
+ if dataformat and dataformat not in ('Twix', 'SPAR', 'Pfile'):
+ return ''
-def get_attribute(dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options: Plugin) -> str:
- """
- This plugin function reads attributes from the supported sourcefile
-
- :param dataformat: The dataformat of the sourcefile, e.g. DICOM of PAR
- :param sourcefile: The sourcefile from which key-value data needs to be read
- :param attribute: The attribute key for which the value needs to be retrieved
- :param options: The bidsmap.plugins['spec2nii2bids'] dictionary with the plugin options
- :return: The retrieved attribute value
- """
-
- if dataformat not in ('Twix', 'SPAR', 'Pfile'):
- return ''
+ suffix = file.suffix.lower()
+ if suffix == '.dat':
+ return 'Twix'
+ elif suffix == '.spar':
+ return 'SPAR'
+ elif suffix == '.7' and not bids.is_dicomfile(file):
+ return 'Pfile'
- if not sourcefile.is_file():
- LOGGER.error(f"Could not find {sourcefile}")
return ''
- if dataformat == 'Twix':
- return bids.get_twixfield(attribute, sourcefile, options.get('multiraid', OPTIONS['multiraid']))
+ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, attribute: str, options: Plugin) -> str:
+ """
+ This plugin function reads attributes from the supported sourcefile
- if dataformat == 'SPAR':
+ :param dataformat: The dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which key-value data needs to be read
+ :param attribute: The attribute key for which the value needs to be retrieved
+ :param options: The bidsmap.plugins['spec2nii2bids'] dictionary with the plugin options
+ :return: The retrieved attribute value
+ """
- return bids.get_sparfield(attribute, sourcefile)
+ if dataformat not in ('Twix', 'SPAR', 'Pfile'):
+ return ''
- if dataformat == 'Pfile':
+ if not sourcefile.is_file():
+ LOGGER.error(f"Could not find {sourcefile}")
+ return ''
- return bids.get_p7field(attribute, sourcefile)
+ if dataformat == 'Twix':
- LOGGER.error(f"Unsupported MRS data-format: {dataformat}")
+ return bids.get_twixfield(attribute, sourcefile, options.get('multiraid', OPTIONS['multiraid']))
+ if dataformat == 'SPAR':
-def bidsmapper_plugin(session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
- """
- The goal of this plugin function is to identify all the different runs in the session and update the
- bidsmap if a new run is discovered
-
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap_new: The new study bidsmap that we are building
- :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
- :param template: The template bidsmap with the default heuristics
- """
+ return bids.get_sparfield(attribute, sourcefile)
- # See for every source file in the session if we already discovered it or not
- for sourcefile in session.rglob('*'):
+ if dataformat == 'Pfile':
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
+ return bids.get_p7field(attribute, sourcefile)
- # See if we can find a matching run in the old bidsmap
- run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+ LOGGER.error(f"Unsupported MRS data-format: {dataformat}")
- # If not, see if we can find a matching run in the template
- if not oldmatch:
- run, _ = template.get_matching_run(sourcefile, dataformat)
+ @due.dcite(Doi('10.1002/mrm.29418'), description='Multi-format in vivo MR spectroscopy conversion to NIFTI', tags=['reference-implementation'])
+ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
+ """
+ This wrapper function around spec2nii converts the MRS data in the session folder and saves it in the bidsfolder.
+ Each saved datafile should be accompanied by a json sidecar file. The bidsmap options for this plugin can be found in:
- # See if we have already put the run somewhere in our new bidsmap
- if not bidsmap_new.exist_run(run):
+ bidsmap.plugins['spec2nii2bids']
- # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
- if not oldmatch:
- LOGGER.info(f"Discovered sample: {run.datasource}")
- else:
- LOGGER.bcdebug(f"Known sample: {run.datasource}")
+ :param session: The full-path name of the subject/session raw data source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
+ """
- # Copy the filled-in run over to the new bidsmap
- bidsmap_new.insert_run(run)
+ # Get started
+ subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
+ sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
+ options = bidsmap.plugins['spec2nii2bids']
+ runid = ''
+ # Read or create a scans_table and tsv-file
+ scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
+ if scans_tsv.is_file():
+ scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
else:
- LOGGER.bcdebug(f"Existing/duplicate sample: {run.datasource}")
-
-
-@due.dcite(Doi('10.1002/mrm.29418'), description='Multi-format in vivo MR spectroscopy conversion to NIFTI', tags=['reference-implementation'])
-def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
- """
- This wrapper function around spec2nii converts the MRS data in the session folder and saves it in the bidsfolder.
- Each saved datafile should be accompanied by a json sidecar file. The bidsmap options for this plugin can be found in:
-
- bidsmap.plugins['spec2nii2bids']
-
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
- :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
- :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
- """
-
- # Get started
- subid = bidsses.name if bidsses.name.startswith('sub-') else bidsses.parent.name
- sesid = bidsses.name if bidsses.name.startswith('ses-') else ''
- options = bidsmap.plugins['spec2nii2bids']
- runid = ''
-
- # Read or create a scans_table and tsv-file
- scans_tsv = bidsses/f"{subid}{'_'+sesid if sesid else ''}_scans.tsv"
- if scans_tsv.is_file():
- scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
- else:
- scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
- scans_table.index.name = 'filename'
-
- # Loop over all MRS source data files and convert them to BIDS
- for sourcefile in session.rglob('*'):
-
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
-
- # Get a matching run from the bidsmap
- run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
-
- # Check if we should ignore this run
- if run.datatype in bidsmap.options['ignoretypes']:
- LOGGER.info(f"--> Leaving out: {run.datasource}")
- bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
- continue
-
- # Check that we know this run
- if not runid:
- LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete the MRS output data in {bidsses} to solve this warning")
- bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
- continue
-
- LOGGER.info(f"--> Coining: {run.datasource}")
-
- # Create the BIDS session/datatype output folder
- outfolder = bidsses/run.datatype
- outfolder.mkdir(parents=True, exist_ok=True)
-
- # Compose the BIDS filename using the matched run
- bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
- bidsname = run.get_bidsname(subid, sesid, not bidsignore, runtime=True)
- bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
- bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
- target = (outfolder/bidsname).with_suffix('.nii.gz')
-
- # Check if the bidsname is valid
- bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
- isbids = BIDSValidator().is_bids(bidstest)
- if not isbids and not bidsignore:
- LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
-
- # Check if file already exists (-> e.g. when a static runindex is used)
- if target.is_file():
- LOGGER.warning(f"{outfolder/bidsname}.* already exists and will be deleted -- check your results carefully!")
- for ext in ('.nii.gz', '.nii', '.json', '.tsv', '.tsv.gz', '.bval', '.bvec'):
- target.with_suffix('').with_suffix(ext).unlink(missing_ok=True)
-
- # Run spec2nii to convert the source-files in the run folder to NIfTI's in the BIDS-folder
- arg = ''
- args = options.get('args', OPTIONS['args']) or ''
- if dataformat == 'SPAR':
- dformat = 'philips'
- arg = f'"{sourcefile.with_suffix(".SDAT")}"'
- elif dataformat == 'Twix':
- dformat = 'twix'
- arg = '-e image'
- elif dataformat == 'Pfile':
- dformat = 'ge'
- else:
- LOGGER.error(f"Unsupported dataformat: {dataformat}")
- return
- command = options.get('command', 'spec2nii')
- errcode = run_command(f'{command} {dformat} -j -f "{bidsname}" -o "{outfolder}" {args} {arg} "{sourcefile}"')
- bids.bidsprov(bidsses, sourcefile, run, [target] if target.is_file() else [])
- if not target.is_file():
- if not errcode:
- LOGGER.error(f"Output file not found: {target}")
- continue
-
- # Load/copy over and adapt the newly produced json sidecar-file
- sidecar = target.with_suffix('').with_suffix('.json')
- metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta',[]))
- if metadata:
- with sidecar.open('w') as json_fid:
- json.dump(metadata, json_fid, indent=4)
-
- # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
- attributes = run.datasource.attributes
- if not bidsignore:
- acq_time = ''
+ scans_table = pd.DataFrame(columns=['acq_time'], dtype='str')
+ scans_table.index.name = 'filename'
+
+ # Loop over all MRS source data files and convert them to BIDS
+ for sourcefile in session.rglob('*'):
+
+ # Check if the sourcefile is of a supported dataformat
+ if is_hidden(sourcefile.relative_to(session)) or not (dataformat := self.has_support(sourcefile)):
+ continue
+
+ # Get a matching run from the bidsmap
+ run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
+
+ # Check if we should ignore this run
+ if run.datatype in bidsmap.options['ignoretypes']:
+ LOGGER.info(f"--> Leaving out: {run.datasource}")
+ bids.bidsprov(bidsses, sourcefile, run) # Write out empty provenance logging data
+ continue
+
+ # Check that we know this run
+ if not runid:
+ LOGGER.error(f"Skipping unknown run: {run.datasource}\n-> Re-run the bidsmapper and delete the MRS output data in {bidsses} to solve this warning")
+ bids.bidsprov(bidsses, sourcefile) # Write out empty provenance logging data
+ continue
+
+ LOGGER.info(f"--> Coining: {run.datasource}")
+
+ # Create the BIDS session/datatype output folder
+ outfolder = bidsses/run.datatype
+ outfolder.mkdir(parents=True, exist_ok=True)
+
+ # Compose the BIDS filename using the matched run
+ bidsignore = bids.check_ignore(run.datatype, bidsmap.options['bidsignore'])
+ bidsname = run.get_bidsname(subid, sesid, not bidsignore, runtime=True)
+ bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap.options['bidsignore'], 'file')
+ bidsname = run.increment_runindex(outfolder, bidsname, scans_table)
+ target = (outfolder/bidsname).with_suffix('.nii.gz')
+
+ # Check if the bidsname is valid
+ bidstest = (Path('/')/subid/sesid/run.datatype/bidsname).with_suffix('.nii').as_posix()
+ isbids = BIDSValidator().is_bids(bidstest)
+ if not isbids and not bidsignore:
+ LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test")
+
+ # Check if file already exists (-> e.g. when a static runindex is used)
+ if target.is_file():
+ LOGGER.warning(f"{outfolder/bidsname}.* already exists and will be deleted -- check your results carefully!")
+ for ext in ('.nii.gz', '.nii', '.json', '.tsv', '.tsv.gz', '.bval', '.bvec'):
+ target.with_suffix('').with_suffix(ext).unlink(missing_ok=True)
+
+ # Run spec2nii to convert the source-files in the run folder to NIfTI's in the BIDS-folder
+ arg = ''
+ args = options.get('args', OPTIONS['args']) or ''
if dataformat == 'SPAR':
- acq_time = attributes('scan_date')
+ dformat = 'philips'
+ arg = f'"{sourcefile.with_suffix(".SDAT")}"'
elif dataformat == 'Twix':
- acq_time = f"{attributes('AcquisitionDate')}T{attributes('AcquisitionTime')}"
+ dformat = 'twix'
+ arg = '-e image'
elif dataformat == 'Pfile':
- acq_time = f"{attributes('rhr_rh_scan_date')}T{attributes('rhr_rh_scan_time')}"
- if not acq_time or acq_time == 'T':
- acq_time = f"1925-01-01T{metadata.get('AcquisitionTime','')}"
+ dformat = 'ge'
+ else:
+ LOGGER.error(f"Unsupported dataformat: {dataformat}")
+ return
+ command = options.get('command', 'spec2nii')
+ errcode = run_command(f'{command} {dformat} -j -f "{bidsname}" -o "{outfolder}" {args} {arg} "{sourcefile}"')
+ bids.bidsprov(bidsses, sourcefile, run, [target] if target.is_file() else [])
+ if not target.is_file():
+ if not errcode:
+ LOGGER.error(f"Output file not found: {target}")
+ continue
+
+ # Load/copy over and adapt the newly produced json sidecar-file
+ sidecar = target.with_suffix('').with_suffix('.json')
+ metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, options.get('meta',[]))
+ if metadata:
+ with sidecar.open('w') as json_fid:
+ json.dump(metadata, json_fid, indent=4)
+
+ # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
+ attributes = run.datasource.attributes
+ if not bidsignore:
+ acq_time = ''
+ if dataformat == 'SPAR':
+ acq_time = attributes('scan_date')
+ elif dataformat == 'Twix':
+ acq_time = f"{attributes('AcquisitionDate')}T{attributes('AcquisitionTime')}"
+ elif dataformat == 'Pfile':
+ acq_time = f"{attributes('rhr_rh_scan_date')}T{attributes('rhr_rh_scan_time')}"
+ if not acq_time or acq_time == 'T':
+ acq_time = f"1925-01-01T{metadata.get('AcquisitionTime','')}"
+ try:
+ acq_time = dateutil.parser.parse(acq_time)
+ if options.get('anon',OPTIONS['anon']) in ('y','yes'):
+ acq_time = acq_time.replace(year=1925, month=1, day=1) # Privacy protection (see BIDS specification)
+ acq_time = acq_time.isoformat()
+ except Exception as jsonerror:
+ LOGGER.warning(f"Could not parse the acquisition time from: {sourcefile}\n{jsonerror}")
+ acq_time = 'n/a'
+ scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time
+
+ if not runid:
+ LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
+ return
+
+ # Write the scans_table to disk
+ LOGGER.verbose(f"Writing acquisition time data to: {scans_tsv}")
+ scans_table.sort_values(by=['acq_time', 'filename'], inplace=True)
+ scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
+
+ # Collect personal data for the participants.tsv file (assumes the dataformat and personal attributes remain the same in the session)
+ personals = {}
+ age = ''
+ if dataformat == 'Twix':
+ personals['sex'] = attributes('PatientSex')
+ personals['size'] = attributes('PatientSize')
+ personals['weight'] = attributes('PatientWeight')
+ age = attributes('PatientAge') # A string of characters with one of the following formats: nnnD, nnnW, nnnM, nnnY
+ elif dataformat == 'Pfile':
+ sex = attributes('rhe_patsex')
+ if sex == '0': personals['sex'] = 'O'
+ elif sex == '1': personals['sex'] = 'M'
+ elif sex == '2': personals['sex'] = 'F'
try:
- acq_time = dateutil.parser.parse(acq_time)
- if options.get('anon',OPTIONS['anon']) in ('y','yes'):
- acq_time = acq_time.replace(year=1925, month=1, day=1) # Privacy protection (see BIDS specification)
- acq_time = acq_time.isoformat()
- except Exception as jsonerror:
- LOGGER.warning(f"Could not parse the acquisition time from: {sourcefile}\n{jsonerror}")
- acq_time = 'n/a'
- scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time
-
- if not runid:
- LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")
- return
-
- # Write the scans_table to disk
- LOGGER.verbose(f"Writing acquisition time data to: {scans_tsv}")
- scans_table.sort_values(by=['acq_time', 'filename'], inplace=True)
- scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
-
- # Collect personal data for the participants.tsv file (assumes the dataformat and personal attributes remain the same in the session)
- personals = {}
- age = ''
- if dataformat == 'Twix':
- personals['sex'] = attributes('PatientSex')
- personals['size'] = attributes('PatientSize')
- personals['weight'] = attributes('PatientWeight')
- age = attributes('PatientAge') # A string of characters with one of the following formats: nnnD, nnnW, nnnM, nnnY
- elif dataformat == 'Pfile':
- sex = attributes('rhe_patsex')
- if sex == '0': personals['sex'] = 'O'
- elif sex == '1': personals['sex'] = 'M'
- elif sex == '2': personals['sex'] = 'F'
+ age = dateutil.parser.parse(attributes('rhr_rh_scan_date')) - dateutil.parser.parse(attributes('rhe_dateofbirth'))
+ age = str(age.days) + 'D'
+ except dateutil.parser.ParserError as exc:
+ pass
try:
- age = dateutil.parser.parse(attributes('rhr_rh_scan_date')) - dateutil.parser.parse(attributes('rhe_dateofbirth'))
- age = str(age.days) + 'D'
- except dateutil.parser.ParserError as exc:
- pass
- try:
- if age.endswith('D'): age = float(age.rstrip('D')) / 365.2524
- elif age.endswith('W'): age = float(age.rstrip('W')) / 52.1775
- elif age.endswith('M'): age = float(age.rstrip('M')) / 12
- elif age.endswith('Y'): age = float(age.rstrip('Y'))
- if age:
- if options.get('anon',OPTIONS['anon']) in ('y','yes'):
- age = int(float(age))
- personals['age'] = str(age)
- except Exception as exc:
- LOGGER.warning(f"Could not parse age from: {run.datasource}\n{exc}")
-
- return personals
+ if age.endswith('D'): age = float(age.rstrip('D')) / 365.2524
+ elif age.endswith('W'): age = float(age.rstrip('W')) / 52.1775
+ elif age.endswith('M'): age = float(age.rstrip('M')) / 12
+ elif age.endswith('Y'): age = float(age.rstrip('Y'))
+ if age:
+ if options.get('anon',OPTIONS['anon']) in ('y','yes'):
+ age = int(float(age))
+ personals['age'] = str(age)
+ except Exception as exc:
+ LOGGER.warning(f"Could not parse age from: {run.datasource}\n{exc}")
+
+ return personals
diff --git a/bidscoin/utilities/dicomsort.py b/bidscoin/utilities/dicomsort.py
index faa0b4e5..4b3b9081 100755
--- a/bidscoin/utilities/dicomsort.py
+++ b/bidscoin/utilities/dicomsort.py
@@ -6,7 +6,6 @@
import uuid
from pydicom import fileset
from pathlib import Path
-from typing import List, Set
from importlib.util import find_spec
from typing import Union
if find_spec('bidscoin') is None:
@@ -81,7 +80,7 @@ def cleanup(name: str) -> str:
return name
-def sortsession(sessionfolder: Path, dicomfiles: List[Path], folderscheme: str, namescheme: str, force: bool, dryrun: bool) -> None:
+def sortsession(sessionfolder: Path, dicomfiles: list[Path], folderscheme: str, namescheme: str, force: bool, dryrun: bool) -> None:
"""
Sorts dicomfiles into subfolders (e.g. a 3-digit SeriesNumber-SeriesDescription subfolder, such as '003-T1MPRAGE')
@@ -136,7 +135,7 @@ def sortsession(sessionfolder: Path, dicomfiles: List[Path], folderscheme: str,
def sortsessions(sourcefolder: Path, subprefix: Union[str,None]='', sesprefix: str='', folderscheme: str='{SeriesNumber:03d}-{SeriesDescription}',
- namescheme: str='', pattern: str=r'.*\.(IMA|dcm)$', recursive: bool=True, force: bool=False, dryrun: bool=False) -> Set[Path]:
+ namescheme: str='', pattern: str=r'.*\.(IMA|dcm)$', recursive: bool=True, force: bool=False, dryrun: bool=False) -> set[Path]:
"""
Wrapper around sortsession() to loop over subjects and sessions and map the session DICOM files
@@ -168,7 +167,7 @@ def sortsessions(sourcefolder: Path, subprefix: Union[str,None]='', sesprefix: s
return set()
# Do a recursive call if a sub- or ses-prefix is given
- sessions: Set[Path] = set() # Collect the sorted session-folders
+ sessions: set[Path] = set() # Collect the sorted session-folders
if subprefix or sesprefix:
LOGGER.info(f"Searching for subject/session folders in: {sourcefolder}")
for subjectfolder in lsdirs(sourcefolder, (subprefix or '') + '*'):
diff --git a/docs/bidsmap.rst b/docs/bidsmap.rst
index 1489943e..d5f7a47e 100644
--- a/docs/bidsmap.rst
+++ b/docs/bidsmap.rst
@@ -11,6 +11,7 @@ A central concept in BIDScoin is the so-called bidsmap. Generally speaking, a bi
3. **The attributes dictionary** contains attributes from the source data itself, such as the 'ProtocolName' from the DICOM header. The source attributes are a very rich source of information of which a minimal subset is normally sufficient to identify the different data types in your source data repository. The attributes are read from (the header of) the source file itself or, if present, from an accompanying sidecar file. This sidecar file transparently extends (or overrule) the available source attributes, as if that data would have been written to (the header of) the source data file itself. The name of the sidecar file should be the same as the name of the first associated source file and have a ``.json`` file extension. For instance, the ``001.dcm``, ``002.dcm``, ``003.dcm``, [..], DICOM source images can have a sidecar file in the same directory named ``001.json`` (e.g. containing metadata that is not available in the DICOM header or that must be overruled). It should be noted that BIDScoin `plugins <./plugins.html>`__ will copy the extended attribute data over to the json sidecar files in your BIDS output folder, giving you additional control to generate your BIDS sidecar files (in addition to the meta dictionary described in point 5 below).
4. **The bids dictionary** contains the BIDS data type and entities that determine the filename of the BIDS output data. The values in this dictionary are encouraged to be edited by the user
5. **The meta dictionary** contains custom key-value pairs that are added to the json sidecar file by the BIDScoin plugins. Meta data may well vary from session to session, hence this dictionary often contains dynamic attribute values that are evaluated during bidscoiner runtime (see the `special features <#special-bidsmap-features>`__ below)
+6. **The events dictionary** contains settings for parsing Events data (WIP))
In sum, a run-item contains a single bids-mapping, which links the input dictionaries (2) and (3) to the output dictionaries (4) and (5).
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 4d867c56..6136efcb 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1,7 +1,7 @@
Plugins
=======
-As shown in the figure below, all interactions of BIDScoin routines with source data are done via a plugin layer that abstracts away differences between source data formats. The bidsmapper and bidscoiner tools loop over the subjects/sessions in your source data repository and then use the plugins that are listed in the bidsmap to do the actual work.
+As shown in the figure below, all interactions of BIDScoin routines with source data are done via an Interface class that abstracts away differences between source data formats. The bidsmapper and bidscoiner tools loop over the subjects/sessions in your source data repository and then use the plugins that are listed in the bidsmap to do the actual work.
.. figure:: ./_static/bidscoin_architecture.png
@@ -39,15 +39,15 @@ This paragraph describes the requirements and structure of plugins in order to a
The main task of a plugin is to perform the actual conversion of the source data into a format that is part of the BIDS standard. BIDScoin offers the Python library module named ``bids`` to interact with bidsmaps and to provide the intended output names and meta data. Notably, the bids library contains a class named ``BidsMap()`` that provides various methods and other useful classes for building and interacting with bidsmap data. Bidsmap objects provide consecutive access to ``DataFormat()``, ``Datatype()``, ``RunItem()`` and ``DataSource()`` objects, each of which comes with methods to interact with the corresponding sections of the bidsmap data. The RunItem objects can be used to obtain the mapping to the BIDS output names, and the DataSource object can read the source data attributes and properties. The DataSource object transparently handles dynamic values (including regular expressions) as well as the extended source data attributes.
-In short, the purpose of the plugin is to interact with the data, by providing these methods:
+In short, the purpose of the plugin is to interact with the data, by providing methods from the abstract base class ``bidscoin.plugins.PluginInterface``. Most notably, plugins can implement the following methods:
-- **test()**: A test function for the plugin + its bidsmap options. Can be called by the user from the bidseditor and the bidscoin utility
+- **test()**: Optional. A test function for the plugin + its bidsmap options. Can be called by the user from the bidseditor and the bidscoin utility
- **has_support()**: If given a source data file that the plugin supports, then report back the name of its data format, i.e. the name of the section in the bidsmap
- **get_attribute()**: If given a source data file that the plugin supports, then report back its attribute value (e.g. from the header)
-- **bidsmapper_plugin()**: From a given session folder, identify the different runs (source datatypes) and, if they haven't been discovered yet, add them to the study bidsmap
-- **bidscoiner_plugin()**: From a given session folder, identify the different runs (source datatypes) and convert them to BIDS output files using the mapping data specified in the runitem
+- **bidsmapper()**: Optional. From a given session folder, identify the different runs (source datatypes) and, if they haven't been discovered yet, add them to the study bidsmap
+- **bidscoiner()**: From a given session folder, identify the different runs (source datatypes) and convert them to BIDS output files using the mapping data specified in the runitem
-Optionally, a ``EventsParser()`` class can be defined to convert stimulus presentation log data to task events files. This class inherits from the equally named class in the ``bids`` library, and should add code to make an initial parsing of the source data to a Pandas DataFrame (table).
+In addition, a class named ``[DataFormat]Events()`` can be added to convert stimulus presentation log data to task events files. This class inherits from ``bidscoin.plugins.EventsParser``, and must implement code to make an initial parsing of the source data to a Pandas DataFrame (table).
The above API is illustrated in more detail in the placeholder Python code below. For real world examples you best first take a look at the nibabel2bids plugin, which exemplifies a clean and fairly minimal implementation of the required functionality. A similar, but somewhat more elaborated implementation (supporting multiple dataformats) can be found in the spec2nii2bids plugin. Finally, the dcm2niix2bids plugin is the more complicated example, due to the logic needed to deal with special output files and various irregularities.
@@ -60,194 +60,83 @@ The above API is illustrated in more detail in the placeholder Python code below
LOGGER = logging.getLogger(__name__)
- # The default options that are set when installing the plugin. This is optional and acts as a fallback
- # in case the plugin options are not specified in the (template) bidsmap
- OPTIONS = {'command': 'demo', # Plugin option
- 'args': 'foo bar'} # Another plugin option
-
- # The default bids-mappings that are added when installing the plugin. This is optional and only acts
- # as a fallback in case the dataformat section is not present in the bidsmap. So far, this feature is
- # not used by any of the plugins
- BIDSMAP = {'DemoFormat':{
- 'subject': '<>', # This filesystem property extracts the subject label from the source directory. NB: Any property or attribute can be used, e.g.
- 'session': '<>', # This filesystem property extracts the session label from the source directory. NB: Any property or attribute can be used, e.g.
-
- 'func': [ # ----------------------- All functional runs --------------------
- {'provenance': '', # The fullpath name of the source file from which the attributes and properties are read. Serves also as a look-up key to find a run in the bidsmap
- 'properties': # The matching (regex) criteria go in here
- {'filepath': '', # File folder, e.g. ".*Parkinson.*" or ".*(phantom|bottle).*"
- 'filename': '', # File name, e.g. ".*fmap.*" or ".*(fmap|field.?map|B0.?map).*"
- 'filesize': '', # File size, e.g. "2[4-6]\d MB" for matching files between 240-269 MB
- 'nrfiles': ''}, # Number of files in the folder that match the above criteria, e.g. "5/d/d" for matching a number between 500-599
- 'attributes': # The matching (regex) criteria go in here
- {'ch_num': '.*',
- 'filetype': '.*',
- 'freq': '.*',
- 'ch_name': '.*',
- 'units': '.*',
- 'trigger_idx': '.*'},
- 'bids':
- {'task': '',
- 'acq': '',
- 'ce': '',
- 'dir': '',
- 'rec': '',
- 'run': '<<>>', # This will be updated during bidscoiner runtime (as it depends on the already existing files)
- 'recording': '',
- 'suffix': 'physio'},
- 'meta': # This is an optional entry for meta-data dictionary that are appended to the json sidecar files
- {'TriggerChannel': '<>',
- 'TimeOffset': '<>'}}],
-
- 'exclude': [ # ----------------------- Data that will be left out -------------
- {'attributes':
- {'ch_num': '.*',
- 'filetype': '.*',
- 'freq': '.*',
- 'ch_name': '.*',
- 'units': '.*',
- 'trigger_idx': '.*'},
- 'bids':
- {'task': '',
- 'acq': '',
- 'ce': '',
- 'dir': '',
- 'rec': '',
- 'run': '<<>>',
- 'recording': '',
- 'suffix': 'physio'}
-
-
- def test(options: dict=OPTIONS) -> int:
- """
- Performs a runtime/integration test of the working of this plugin + given options
-
- :param options: A dictionary with the plugin options, e.g. taken from `bidsmap.plugins[__name__]`
- :return: The errorcode (e.g 0 if the tool generated the expected result, > 0 if there was
- a tool error)
- """
-
- LOGGER.info(f'This is a demo-plugin test routine, validating its working with options: {options}')
-
- return 0
-
-
- def has_support(file: Path) -> str:
- """
- This plugin function assesses whether a sourcefile is of a supported dataformat
-
- :param file: The sourcefile that is assessed
- :param dataformat: The requested dataformat (optional requirement)
- :return: The name of the supported dataformat of the sourcefile. This name should
- correspond to the name of a dataformat in the bidsmap
- """
-
- if file.is_file():
-
- LOGGER.verbose(f'This has_support routine assesses whether "{file}" is of a known dataformat')
- return 'dataformat_name' if file == 'of_a_supported_format' else ''
-
- return ''
-
-
- def get_attribute(dataformat: str, sourcefile: Path, attribute: str, options: dict) -> str:
- """
- This plugin function reads attributes from the supported sourcefile
-
- :param dataformat: The dataformat of the sourcefile, e.g. DICOM of PAR
- :param sourcefile: The sourcefile from which key-value data needs to be read
- :param attribute: The attribute key for which the value needs to be retrieved
- :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins[__name__]
- :return: The retrieved attribute value
- """
-
- if dataformat in ('DICOM','PAR'):
- LOGGER.verbose(f'This is a demo-plugin get_attribute routine, reading the {dataformat} "{attribute}" attribute value from "{sourcefile}"')
- return read(sourcefile, attribute)
-
- return ''
-
-
- def bidsmapper_plugin(session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
- """
- The goal of this plugin function is to identify all the different runs in the session and update the
- bidsmap if a new run is discovered
-
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap_new: The new study bidsmap that we are building
- :param bidsmap_old: The previous study bidsmap that has precedence over the template bidsmap
- :param template: The template bidsmap with the default heuristics
- """
-
- # See for every data source in the session if we already discovered it or not
- for sourcefile in session.rglob('*'):
-
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
+ class Interface(PluginInterface):
- # See if we can find a matching run in the old bidsmap
- run, oldmatch = bidsmap_old.get_matching_run(sourcefile, dataformat)
+ def has_support(self, file: Path) -> str:
+ """
+ This plugin function assesses whether a sourcefile is of a supported dataformat
+
+ :param file: The sourcefile that is assessed
+ :param dataformat: The requested dataformat (optional requirement)
+ :return: The name of the supported dataformat of the sourcefile. This name should
+ correspond to the name of a dataformat in the bidsmap
+ """
+
+ if file.is_file():
- # If not, see if we can find a matching run in the template
- if not oldmatch:
- run, _ = template.get_matching_run(sourcefile, dataformat)
+ LOGGER.verbose(f'This has_support routine assesses whether "{file}" is of a known dataformat')
+ return 'dataformat_name' if file == 'of_a_supported_format' else ''
- # See if we have already put the run somewhere in our new bidsmap
- if not bidsmap_new.exist_run(run):
+ return ''
- # Communicate with the user if the run was not present in bidsmap_old or in template, i.e. that we found a new sample
- if not oldmatch:
- LOGGER.info(f"Discovered sample: {run.datasource}")
+ def get_attribute(self, dataformat: str, sourcefile: Path, attribute: str, options: dict) -> str:
+ """
+ This plugin function reads attributes from the supported sourcefile
- # Do some stuff with the run if needed
- pass
+ :param dataformat: The dataformat of the sourcefile, e.g. DICOM of PAR
+ :param sourcefile: The sourcefile from which key-value data needs to be read
+ :param attribute: The attribute key for which the value needs to be retrieved
+ :param options: A dictionary with the plugin options, e.g. taken from the bidsmap.plugins[__name__]
+ :return: The retrieved attribute value
+ """
- # Copy the filled-in run over to the new bidsmap
- bidsmap_new.insert_run(run)
+ if dataformat in ('DICOM','PAR'):
+ LOGGER.verbose(f'This is a demo-plugin get_attribute routine, reading the {dataformat} "{attribute}" attribute value from "{sourcefile}"')
+ return read(sourcefile, attribute)
+ return ''
- @due.dcite(Doi('put.your/doi.here'), description='This is an optional duecredit decorator for citing your paper(s)', tags=['implementation'])
- def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
- """
- The plugin to convert the runs in the source folder and save them in the bids folder. Each saved datafile should be
- accompanied by a json sidecar file. The bidsmap options for this plugin can be found in:
+ @due.dcite(Doi('put.your/doi.here'), description='This is an optional duecredit decorator for citing your paper(s)', tags=['implementation'])
+ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[None, dict]:
+ """
+ The plugin to convert the runs in the source folder and save them in the bids folder. Each saved datafile should be
+ accompanied by a json sidecar file. The bidsmap options for this plugin can be found in:
- bidsmap.plugins[__name__]
+ bidsmap.plugins[__name__]
- See also the dcm2niix2bids plugin for reference implementation
+ See also the dcm2niix2bids plugin for reference implementation
- :param session: The full-path name of the subject/session raw data source folder
- :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
- :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
- :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
- """
+ :param session: The full-path name of the subject/session raw data source folder
+ :param bidsmap: The full mapping heuristics from the bidsmap YAML-file
+ :param bidsses: The full-path name of the BIDS output `sub-/ses-` folder
+ :return: A dictionary with personal data for the participants.tsv file (such as sex or age)
+ """
- # Go over the different source files in the session
- for sourcefile in session.rglob('*'):
+ # Go over the different source files in the session
+ for sourcefile in session.rglob('*'):
- # Check if the sourcefile is of a supported dataformat
- if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
- continue
+ # Check if the sourcefile is of a supported dataformat
+ if is_hidden(sourcefile.relative_to(session)) or not (dataformat := has_support(sourcefile)):
+ continue
- # Get a matching run from the bidsmap
- run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
+ # Get a matching run from the bidsmap
+ run, runid = bidsmap.get_matching_run(sourcefile, dataformat, runtime=True)
- # Compose the BIDS filename using the matched run
- bidsname = run.bidsname(subid, sesid, validkeys=True, runtime=True)
+ # Compose the BIDS filename using the matched run
+ bidsname = run.bidsname(subid, sesid, validkeys=True, runtime=True)
- # Save the sourcefile as a BIDS NIfTI file
- targetfile = (outfolder/bidsname).with_suffix('.nii')
- convert(sourcefile, targetfile)
+ # Save the sourcefile as a BIDS NIfTI file
+ targetfile = (outfolder/bidsname).with_suffix('.nii')
+ convert(sourcefile, targetfile)
- # Write out provenance logging data (= useful but not strictly necessary)
- bids.bidsprov(bidsses, sourcefile, run, targetfile)
+ # Write out provenance logging data (= useful but not strictly necessary)
+ bids.bidsprov(bidsses, sourcefile, run, targetfile)
- # Pool all sources of meta-data and save it as a json sidecar file
- sidecar = targetfile.with_suffix('.json')
- ext_meta = bidsmap.plugins[__name__]['meta']
- metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, ext_meta)
- save(sidecar, metadata)
+ # Pool all sources of meta-data and save it as a json sidecar file
+ sidecar = targetfile.with_suffix('.json')
+ ext_meta = bidsmap.plugins[__name__]['meta']
+ metadata = bids.poolmetadata(run.datasource, sidecar, run.meta, ext_meta)
+ save(sidecar, metadata)
class PresentationEvents(EventsParser):
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 4b7f4771..e87ecfa0 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -18,12 +18,12 @@
def test_plugin(plugin, options):
# First test to see if we can import the plugin
- module = bcoin.import_plugin(plugin, ('bidsmapper_plugin', 'bidscoiner_plugin'))
+ module = bcoin.import_plugin(plugin)
if not inspect.ismodule(module):
raise ImportError(f"Invalid plugin: '{plugin}'")
# Then run the plugin's own 'test' routine (if implemented)
- assert module.test(options.get(plugin.stem, {})) == 0
+ assert module.Interface().test(options.get(plugin.stem, {})) == 0
# Test that we don't import invalid plugins
module = bcoin.import_plugin(plugin, ('foo_plugin', 'bar_plugin'))