diff --git a/bidscoin/bids.py b/bidscoin/bids.py index 7ef9a9ee..b540f9cd 100644 --- a/bidscoin/bids.py +++ b/bidscoin/bids.py @@ -985,9 +985,9 @@ def load_bidsmap(yamlfile: Path=Path(), folder: Path=templatefolder, plugins:Ite for dataformat in bidsmap: if dataformat in ('$schema', 'Options'): continue bidsmap[dataformat]['session'] = bidsmap[dataformat]['session'] or '' # Session-less data repositories - for datatype in bidsmap[dataformat]: - if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject', 'session' and empty datatypes - for index, run in enumerate(bidsmap[dataformat][datatype]): + for datatype in bidsmap[dataformat] or []: + if datatype in ('subject', 'session'): continue + for index, run in enumerate(bidsmap[dataformat][datatype] or []): # Add missing provenance info if not run.get('provenance'): @@ -1045,12 +1045,11 @@ def save_bidsmap(filename: Path, bidsmap: Bidsmap) -> None: :return: """ - # Remove the added DataSource object + # Remove the added DataSource objects bidsmap = copy.deepcopy(bidsmap) for dataformat in bidsmap: if dataformat in ('$schema', 'Options'): continue - if not bidsmap[dataformat]: continue - for datatype in bidsmap[dataformat]: + for datatype in bidsmap[dataformat] or []: if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session' for run in bidsmap[dataformat][datatype]: run.pop('datasource', None) @@ -1091,8 +1090,7 @@ def validate_bidsmap(bidsmap: Bidsmap, level: int=1) -> bool: LOGGER.info(f"bids-validator {bids_validator.__version__} test results (* = in .bidsignore):") for dataformat in bidsmap: if dataformat in ('$schema', 'Options'): continue - if not bidsmap[dataformat]: continue - for datatype in bidsmap[dataformat]: + for datatype in bidsmap[dataformat] or []: if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session' for run in bidsmap[dataformat][datatype]: bidsname = get_bidsname(f"sub-{sanitize(dataformat)}", '', run, False) @@ -1135,8 +1133,7 @@ def check_bidsmap(bidsmap: Bidsmap, checks: Tuple[bool, bool, bool]=(True, True, LOGGER.info('Checking the bidsmap run-items:') for dataformat in bidsmap: if dataformat in ('$schema', 'Options'): continue # TODO: Check Options - if not bidsmap[dataformat]: continue - for datatype in bidsmap[dataformat]: + for datatype in bidsmap[dataformat] or []: if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session' if datatype in bidsmap['Options']['bidscoin']['ignoretypes']: continue # E.g. 'exclude' if check_ignore(datatype, bidsmap['Options']['bidscoin']['bidsignore']): continue @@ -1178,7 +1175,7 @@ def check_template(bidsmap: Bidsmap) -> bool: LOGGER.verbose('Checking the template bidsmap datatypes:') for dataformat in bidsmap: if dataformat in ('$schema', 'Options'): continue - for datatype in bidsmap[dataformat]: + for datatype in bidsmap[dataformat] or []: if not isinstance(bidsmap[dataformat][datatype], list): continue # Skip datatype = 'subject'/'session' if not (datatype in bidsdatatypesdef or datatype in ignoretypes or check_ignore(datatype, bidsignore)): LOGGER.warning(f"Invalid {dataformat} datatype: '{datatype}' (you may want to add it to the 'bidsignore' list)") @@ -1427,7 +1424,7 @@ def create_run(datasource: DataSource=None, bidsmap: Bidsmap=None) -> Run: return Run(dict(provenance = str(datasource.path), properties = {'filepath':'', 'filename':'', 'filesize':'', 'nrfiles':None}, attributes = {}, - bids = {}, + bids = {'suffix':''}, meta = {}, datasource = datasource)) @@ -1518,7 +1515,7 @@ def find_run(bidsmap: Bidsmap, provenance: str, dataformat: str='', datatype: st return Run({}) -def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', dataformat: str='') -> None: +def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', dataformat: str='') -> bool: """ Delete the first matching run from the BIDS map @@ -1526,13 +1523,13 @@ def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', :param provenance: The provenance identifier of/or the run-item that is deleted :param datatype: The datatype that of the deleted run_item (can be different from run_item['datasource']), e.g. 'anat' :param dataformat: The dataformat section in the bidsmap in which the run is deleted, e.g. 'DICOM' - :return: + :return: True if successful, False otherwise """ if isinstance(provenance, str): run_item = find_run(bidsmap, provenance, dataformat) if not run_item: - return + return False else: run_item = provenance provenance = run_item['provenance'] @@ -1545,17 +1542,19 @@ def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', for index, run in enumerate(bidsmap[dataformat].get(datatype,[])): if Path(run['provenance']) == Path(provenance): del bidsmap[dataformat][datatype][index] - return + return True LOGGER.error(f"Could not find (and delete) this [{dataformat}][{datatype}] run: '{provenance}") + return False -def append_run(bidsmap: Bidsmap, run: Run) -> None: +def insert_run(bidsmap: Bidsmap, run: Run, position: int=None) -> None: """ - Append a cleaned-up run to the BIDS map + Inserts a cleaned-up run to the BIDS map :param bidsmap: Full bidsmap data structure, with all options, BIDS labels and attributes, etc. :param run: The run (listitem) that is appended to the datatype + :param position: The position at which the run is inserted. The run is appended at the end if position is None :return: """ @@ -1574,7 +1573,7 @@ def append_run(bidsmap: Bidsmap, run: Run) -> None: if not bidsmap.get(dataformat).get(datatype): bidsmap[dataformat][datatype] = [run] else: - bidsmap[dataformat][datatype].append(run) + bidsmap[dataformat][datatype].insert(len(bidsmap[dataformat][datatype]) if position is None else position, run) def update_bidsmap(bidsmap: Bidsmap, source_datatype: str, run: Run) -> None: @@ -1610,7 +1609,7 @@ def update_bidsmap(bidsmap: Bidsmap, source_datatype: str, run: Run) -> None: delete_run(bidsmap, run, source_datatype) # Append the (cleaned-up) target run - append_run(bidsmap, run) + insert_run(bidsmap, run) else: for index, run_ in enumerate(bidsmap[dataformat][run_datatype]): diff --git a/bidscoin/bidscoiner.py b/bidscoin/bidscoiner.py index 02d5a7c5..27abc59c 100755 --- a/bidscoin/bidscoiner.py +++ b/bidscoin/bidscoiner.py @@ -270,11 +270,11 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force: subid, sesid = datasource.subid_sesid(subid, sesid or '') bidssession = bidsfolder/subid/sesid # TODO: Support DICOMDIR with multiple subjects (as in PYDICOMDIR) if not force and bidssession.is_dir(): - datatypes = [] + datatypes = set() for dataformat in dataformats: for datatype in lsdirs(bidssession): # See what datatypes we already have in the bids session-folder if list(datatype.iterdir()) and bidsmap[dataformat].get(datatype.name): # See if we are going to add data for this datatype - datatypes.append(datatype.name) + datatypes.add(datatype.name) if datatypes: LOGGER.info(f">>> Skipping processed session: {bidssession} already has {datatypes} data (you can carefully use the -f option to overrule)") continue diff --git a/bidscoin/bidsmapper.py b/bidscoin/bidsmapper.py index e13f4e70..788522cf 100755 --- a/bidscoin/bidsmapper.py +++ b/bidscoin/bidsmapper.py @@ -91,7 +91,7 @@ def bidsmapper(sourcefolder: str, bidsfolder: str, bidsmap: str, template: str, unzip = bidsmap_new['Options']['bidscoin'].get('unzip','') for dataformat in bidsmap_new: if dataformat in ('$schema', 'Options'): continue - for datatype in bidsmap_new[dataformat]: + for datatype in bidsmap_new[dataformat] or []: if datatype not in ('subject', 'session'): bidsmap_new[dataformat][datatype] = [] @@ -248,7 +248,7 @@ def main(): trackusage('bidsmapper') try: - bidsmapper(**args(args)) + bidsmapper(**vars(args)) except Exception: trackusage('bidsmapper_exception') diff --git a/bidscoin/heuristics/schema.json b/bidscoin/heuristics/schema.json index 33257633..7ba497fa 100644 --- a/bidscoin/heuristics/schema.json +++ b/bidscoin/heuristics/schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "http://bidscoin.templates/schema.json", + "$id": "file://bidscoin/heuristics/schema.json", "title": "Schema for validating template bidsmaps", "type": "object", "required": ["Options"], @@ -53,8 +53,14 @@ "required": ["subject", "session"], "properties": { - "subject": { "type": "string" }, - "session": { "type": ["string", "integer", "null"] } + "subject": { + "type": "string", + "description": "The participant label (typically a dynamic value)" + }, + "session": { + "type": ["string", "integer", "null"], + "description": "The session label (typically a dynamic value)" + } }, "additionalProperties": { @@ -63,7 +69,7 @@ "items": { "description": "Run-item (containing the bidsmappings)", "type": "object", - "required": ["bids"], + "required": ["provenance", "bids"], "properties": { "provenance": { "description": "The fullpath name of the data source. Serves also as a unique identifier to find a run-item in the bidsmap", diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index 899c0708..66940757 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -196,7 +196,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap, run['datasource'].path = targetfile # Copy the filled-in run over to the new bidsmap - bids.append_run(bidsmap_new, run) + bids.insert_run(bidsmap_new, run) else: LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {dataformat} sample: {sourcefile}") @@ -372,7 +372,7 @@ def bidscoiner_plugin(session: Path, bidsmap: Bidsmap, bidsses: Path) -> Union[N dcm2niixpostfixes = ('_c', '_i', '_Eq', '_real', '_imaginary', '_MoCo', '_t', '_Tilt', '_e', '_ph', '_ADC', '_fieldmaphz') #_c%d, _e%d and _ph (and any combination of these in that order) are for multi-coil data, multi-echo data and phase data dcm2niixfiles = sorted(set([dcm2niixfile for dcm2niixpostfix in dcm2niixpostfixes for dcm2niixfile in outfolder.glob(f"{bidsname}*{dcm2niixpostfix}*.nii*")])) dcm2niixfiles = [dcm2niixfile for dcm2niixfile in dcm2niixfiles if not (re.match(r'sub-.*_echo-[0-9]*\.nii', dcm2niixfile.name) or - re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))] # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see Github issue #232) + re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))] # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see GitHub issue #232) # Rename all dcm2niix files that got additional postfixes (i.e. store the postfixes in the bidsname) for dcm2niixfile in dcm2niixfiles: diff --git a/bidscoin/plugins/nibabel2bids.py b/bidscoin/plugins/nibabel2bids.py index 4d217a95..eb293001 100644 --- a/bidscoin/plugins/nibabel2bids.py +++ b/bidscoin/plugins/nibabel2bids.py @@ -152,7 +152,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap, run['datasource'].path = targetfile # Copy the filled-in run over to the new bidsmap - bids.append_run(bidsmap_new, run) + bids.insert_run(bidsmap_new, run) else: LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {datasource.dataformat} sample: {sourcefile}") diff --git a/bidscoin/plugins/spec2nii2bids.py b/bidscoin/plugins/spec2nii2bids.py index 0f292c01..9dfee1b3 100644 --- a/bidscoin/plugins/spec2nii2bids.py +++ b/bidscoin/plugins/spec2nii2bids.py @@ -157,7 +157,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap, run['datasource'].path = targetfile # Copy the filled-in run over to the new bidsmap - bids.append_run(bidsmap_new, run) + bids.insert_run(bidsmap_new, run) else: LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {dataformat} sample: {sourcefile}") diff --git a/docs/preparation.rst b/docs/preparation.rst index b4f83641..4df1bd4d 100644 --- a/docs/preparation.rst +++ b/docs/preparation.rst @@ -4,7 +4,10 @@ Data organization Supported source data structures -------------------------------- -Out of the box, BIDScoin requires that the source data repository is organized according to a ``subject/[session]/data`` structure (the ``session`` subfolder is always optional). The ``data`` folder(s) can be structured in various ways (depending on the plugin and/or dataformat), as illustrated by the following examples: +Out of the box, BIDScoin requires that the source data repository is organized according to a ``subject/[session]/data`` structure (the ``session`` subfolder is always optional). The ``data`` folder(s) can be structured in various ways (depending on the plugin and/or dataformat), as illustrated by in the sections below. + +.. note:: + You can store your session data as zipped (``.zip``) or tarzipped (e.g. ``.tar.gz``) archive files. BIDScoin `workflow tools <./workflow.html>`__ will automatically unpack/unzip those archive files in a temporary folder and then process your session data from there. For flat/DICOMDIR data, BIDScoin tools (i.e. the bidsmapper and the bidscoiner) will automatically run `dicomsort <./utilities.html#dicomsort>`__ in a temporary folder to sort them in seriesfolders. Depending on the data and file system, repeatedly unzipping data in the workflow may come with a significant processing speed penalty. BIDScoin plugins will skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character. 1. A DICOM Series layout ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -142,12 +145,6 @@ The above layouts are supported by the (default) `dcm2niix2bids <./plugins.html# | [..] [..] -.. note:: - You can store your session data in any of the above data layouts as zipped (``.zip``) or tarzipped (e.g. ``.tar.gz``) archive files. BIDScoin `workflow tools <./workflow.html>`__ will automatically unpack/unzip those archive files in a temporary folder and then process your session data from there. For flat/DICOMDIR data, BIDScoin tools (i.e. the bidsmapper and the bidscoiner) will automatically run `dicomsort <./utilities.html#dicomsort>`__ in a temporary folder to sort them in seriesfolders. Depending on the data and file system, repeatedly unzipping data in the workflow may come with a significant processing speed penalty. - -.. tip:: - BIDScoin plugins will typically skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character. You can use this feature to flexibly omit subjects, sessions or runs from your bids repository, for instance when you restarted an MRI scan because something went wrong with the stimulus presentation and you don't want that data to be converted and enumerated as ``run-1``, ``run-2``. - Recommended data acquisition conventions ---------------------------------------- diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 8875d09b..04311309 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -3,7 +3,7 @@ Troubleshooting Installation ------------ -A first step when encountering execution errors is to test whether your installation is working correctly. An easy way to test the working of various BIDScoin components is to run ``bidscoin -t`` in your terminal. Some commonly seen messages are: +When encountering execution errors it is helpful to know whether your BIDScoin installation is working correctly. An easy way to test the working of its various components is to run ``bidscoin -t`` in your terminal. Some commonly reported issues are: The "dcm2niix" command is not recognized ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -41,7 +41,7 @@ The fix comes from these resources: Workflow -------- -The first step in troubleshooting is to look at the warnings and messages printed out in the terminal (they are also save to disk in the ``bidsfolder/code/bidscoin`` output folder). Make sure you are OK with the warnings (they are meaningful and not to be ignored) and do not continue with a next step until all errors are resolved. +The first step in workflow troubleshooting is to look at the warnings and messages printed out in the terminal (they are also saved to disk in the ``bidsfolder/code/bidscoin`` output folder). Make sure you are OK with the warnings (they are meaningful and not to be ignored) and do not continue with a next step until all errors are resolved. Below you can find answers to some commonly reported issues. My bidsmap is empty ^^^^^^^^^^^^^^^^^^^ @@ -105,6 +105,10 @@ My source-files can no longer be found ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You may get the warning "Cannot reliably change the data type and/or suffix because the source file '..' can no longer be found". This warning is generated when (1) your source data moved to a different location, or (2) your data is zipped or in DICOMDIR format. This warning can be ignored if you do not need to change the data type of your run-items anymore (in the bidseditor), because in that case BIDScoin may need access to the source data (to read new properties or attributes). To restore data access for (1), move the data to it's original location and for (2) use the ``--store`` option of bidsmapper to store local copies of the source data samples in the bids output folder. +Some acquisitions went wrong and need to be excluded +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +BIDScoin plugins will skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character. You can use this feature to flexibly omit subjects, sessions or runs from your bids repository, for instance when you restarted an MRI scan because something went wrong with the stimulus presentation and you don't want that data to be converted and enumerated as ``run-1``, ``run-2``. + I have duplicated field maps because of an interrupted session ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It may happen that due to irregularities during data acquisition you had to reacquire your field-map for part of your data. In that case the ``IntendedFor`` and ``B0FieldIdentifier``/``B0FieldSource`` semantics become ambiguous. To handle this situation, you can use json sidecar files to extend the source attributes (see below) or use the limited ``IntendedFor`` search as described `here <./bidsmap.html#intendedfor>`__ and `here `__. @@ -136,4 +140,4 @@ You can simply use the ``bidseditor`` to make changes to your bidsmap, delete al More help --------- -If this guide does not help to solve your problem, then you can `search on github `__ for open and/or closed issues to see if anyone else has encountered similar problems before. If not, feel free to help yourself and others by opening a new github issue. +If this guide does not help to solve your problem, then you can `search on github `__ for open and/or closed issues to see if anyone else has encountered similar problems before. If not, feel free to help yourself and others by opening a new github issue or post your question on `NeuroStars `__ (tag: #bidscoin)