Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX] Properly handle XNAT-shared experiments #368

Merged
merged 9 commits into from
Jan 17, 2025
Merged

[FIX] Properly handle XNAT-shared experiments #368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dc4b56b
[FIX] Ensure shared experiments use name associated with current project
DESm1th Jan 14, 2025
30d66ed
[REF] Delete SharedExporter (sigh)
DESm1th Jan 15, 2025
0288fd0
[REF] Remove references to shared sessions
DESm1th Jan 15, 2025
807bbb3
[FIX] Add code to rename downloaded scans for XNAT-shared sessions
DESm1th Jan 16, 2025
a701b47
[FIX] Fix var names
DESm1th Jan 16, 2025
bd13460
[FIX] Fix formatting issues
DESm1th Jan 16, 2025
b8d3bb7
[REF] Re-add more informative log message
DESm1th Jan 17, 2025
a49bc1b
[PEP8] Fix formatting issues
DESm1th Jan 17, 2025
7223124
[FIX] Ensure ID modifications are applied to shared sessions
DESm1th Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions bin/dm_xnat_extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def main():

session = datman.scan.Scan(ident, config, bids_root=args.bids_out)

if xnat_experiment.resource_files and not xnat_experiment.is_shared():
if xnat_experiment.resource_files:
export_resources(session.resource_path, xnat, xnat_experiment,
dry_run=args.dry_run)

Expand Down Expand Up @@ -613,7 +613,6 @@ def make_session_exporters(config, session, experiment, bids_opts=None,
"""
formats = get_session_formats(
bids_opts=bids_opts,
shared=experiment.is_shared(),
ignore_db=ignore_db
)

Expand All @@ -627,28 +626,21 @@ def make_session_exporters(config, session, experiment, bids_opts=None,
return exporters


def get_session_formats(bids_opts=None, shared=False, ignore_db=False):
def get_session_formats(bids_opts=None, ignore_db=False):
"""Get the string identifiers for all session exporters that are needed.

Args:
bids_opts (:obj:`BidsOptions`, optional): dcm2bids settings to be
used if exporting to BIDS format. Defaults to None.
shared (bool, optional): Whether to treat the session as a
shared XNAT experiment. Defaults to False.
ignore_db (bool, optional): If True, datman's QC dashboard will not
be updated. Defaults to False.

Returns:
list: a list of string keys that should be used to make exporters.
"""
formats = []
if shared:
formats.append("shared")
elif bids_opts:
# Only do 'bids' format if not a shared session.
formats.append("bids")

if bids_opts:
formats.append("bids")
formats.append("nii_link")
if not ignore_db:
formats.append("db")
Expand Down
258 changes: 23 additions & 235 deletions datman/exporters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import datman.dashboard
import datman.scan
from datman.exceptions import (UndefinedSetting, DashboardException,
ExportException)
from datman.scanid import (parse, parse_bids_filename, ParseException,
parse_filename, make_filename, KCNIIdentifier)
ConfigException)
from datman.scanid import (parse_bids_filename, ParseException,
make_filename, KCNIIdentifier)
from datman.utils import (run, make_temp_directory, get_extension,
filter_niftis, find_tech_notes, read_blacklist,
get_relative_source, read_json, write_json)
Expand Down Expand Up @@ -858,18 +858,19 @@ def make_scan(self, file_stem):
f"with error: {exc}")
return
if self.experiment.is_shared():
self._make_linked(scan)
source_session = self._get_source_session()
self._make_linked(scan, source_session)
self._add_bids_scan_name(scan, file_stem)
self._add_side_car(scan, file_stem)
self._update_conversion_errors(scan, file_stem)

def _make_linked(self, scan):
def _make_linked(self, scan, source_session):
try:
source_session = datman.dashboard.get_session(self.experiment.name)
source_session = datman.dashboard.get_session(source_session)
except datman.dashboard.DashboardException as exc:
logger.error(
f"Failed to link shared scan {scan} to source "
f"{self.experiment.name}. Reason - {exc}"
f"{source_session}. Reason - {exc}"
)
return
matches = [
Expand All @@ -879,14 +880,28 @@ def _make_linked(self, scan):
]
if not matches or len(matches) > 1:
logger.error(
f"Failed to link shared scan {scan} to {self.experiment.name}."
f"Failed to link shared scan {scan} to {source_session}."
" Reason - Unable to find source scan database record."
)
return

scan.source_id = matches[0].id
scan.save()

def _get_source_session(self):
"""Get the ID of the source experiment for a shared XNATExperiment."""
try:
config = datman.config.config(study=self.experiment.source_name)
except ConfigException:
return self.experiment.source_name

try:
id_map = config.get_key('IdMap')
except UndefinedSetting:
return self.experiment.source_name

return str(datman.scanid.parse(self.experiment.source_name, id_map))

def _add_bids_scan_name(self, scan, dm_stem):
"""Add a bids format file name to a series in the QC database.

Expand Down Expand Up @@ -999,233 +1014,6 @@ def errors_outdated(self, scan, fname):
return False


class SharedExporter(SessionExporter):
"""Export an XNAT 'shared' experiment.
"""

type = "shared"
ext = ".nii.gz"

def __init__(self, config, session, experiment, bids_opts=None, **kwargs):
if not experiment.is_shared():
raise ExportException(
f"Cannot make SharedExporter for {experiment}. "
"XNAT Experiment is not shared."
)

try:
self.source_session = self.find_source_session(config, experiment)
except (ParseException, datman.config.ConfigException) as exc:
raise ExportException(
f"Can't find source data for shared experiment {experiment}. "
f"Reason - {exc}"
)

# The datman-style directories to export
dm_dirs = ['qc_path', 'dcm_path', 'mnc_path', 'nrrd_path']
if not bids_opts:
dm_dirs.append('nii_path')

self.tags = config.get_tags(site=session.site)

super().__init__(config, session, experiment, **kwargs)

self.name_map = self.make_name_map(dm_dirs, use_bids=bids_opts)

def find_source_session(self, config, experiment):
"""Find the original data on the filesystem.

Args:
config (:obj:`datman.config.config`): The datman config object for
the study that the shared experiment belongs to.
experiment (:obj:`datman.xnat.XNATExperiment`): The experiment
object for the shared session on XNAT.

Returns:
:obj:`datman.scan.Scan`: The scan object for the source dataset
as previously exported to the filesystem.
"""
ident = parse(experiment.name)
study = config.map_xnat_archive_to_project(ident)
config = datman.config.config(study=study)
return datman.scan.Scan(ident, config)

def make_name_map(self, dm_dirs, use_bids=False):
"""Create a dictionary of source files to their 'shared' alias.

Args:
dm_dirs (:obj:`list`): A list of datman-style paths on the source
session's :obj:`datman.scan.Scan` object to search for files.
use_bids (any, optional): Whether or not to search for bids files.
Any input equivalent to boolean True will be taken as 'True'.
Default False.

Returns:
dict: A dictionary mapping the full path to each discovered source
file to the full path to the output alias name/symlink.
"""
if use_bids:
name_map = self.find_bids_files()
else:
name_map = {}

for dir_type in dm_dirs:
self.find_dm_files(dir_type, name_map)

self.find_resource_files(name_map)

return name_map

def find_bids_files(self, name_map=None):
"""Find all bids files that have been created for the source session.

Args:
name_map (dict, optional): A dictionary that may contain other
discovered files and their output names. Default None.

Returns:
dict: A dictionary of source session files that have been
found, mapped to their full path under the shared/alias ID.
"""
if name_map is None:
name_map = {}

for root, _, files in os.walk(self.source_session.bids_path):
dest_dir = root.replace(
self.source_session.bids_path,
self.session.bids_path
)

for item in files:
dest_name = item.replace(
self.source_session.bids_sub, self.session.bids_sub
).replace(
self.source_session.bids_ses, self.session.bids_ses
)
name_map[os.path.join(root, item)] = os.path.join(
dest_dir, dest_name
)

return name_map

def find_dm_files(self, dir_type, name_map=None):
"""Find datman-style source files in all listed directory types.

Args:
dir_type (list): A list of paths on the source session to search
through. All entries should be valid path types defined for
the `datman.scan.Scan` object.
name_map (dict, optional): A dictionary of other discovered
source files mapped to their aliases. Default None.

Returns:
dict: A dictionary of source session files that have been
found, mapped to their full path under the shared/alias ID.
"""
if name_map is None:
name_map = {}

source_dir = getattr(self.source_session, dir_type)
dest_dir = getattr(self.session, dir_type)

for item in glob(os.path.join(source_dir, "*")):
try:
_, tag, _, _ = parse_filename(item)
except ParseException:
logger.debug(
f"Ignoring invalid file name {item} in {source_dir}"
)
continue

if tag not in self.tags:
# Found a valid scan name but with a tag not used by dest study
continue

if dir_type == 'qc_path' and item.endswith('_manifest.json'):
# Filter out manifest files. These should be regenerated
# by dm_qc_report for the dest session.
continue

fname = os.path.basename(item).replace(
self.source_session.id_plus_session,
self.session.id_plus_session
)
name_map[item] = os.path.join(dest_dir, fname)

return name_map

def find_resource_files(self, name_map=None):
"""Find all source session resources files.

Args:
name_map (dict, optional): A dictionary of any previously found
source files mapped to their aliases.

Returns:
dict: A dictionary of source session files that have been
found, mapped to their full path under the shared/alias ID.
"""
if name_map is None:
name_map = {}

for root, _, files in os.walk(self.session.resource_path):
dest_path = root.replace(
self.source_session.resource_path,
self.session.resource_path
)
for item in files:
name_map[os.path.join(root, item)] = os.path.join(
dest_path, item
)

return name_map

def export(self, *args, **kwargs):
if self.dry_run:
logger.info(
"Dry run: Skipping export of shared session files "
f"{self.name_map}"
)
return

for source in self.name_map:
parent, _ = os.path.split(self.name_map[source])
try:
os.makedirs(parent)
except FileExistsError:
pass
except PermissionError:
logger.error(
f"Failed to make dir {parent} for session {self.session}. "
"Permission denied."
)
continue
rel_source = get_relative_source(source, self.name_map[source])
try:
os.symlink(rel_source, self.name_map[source])
except FileExistsError:
pass
except PermissionError:
logger.error(
f"Failed to create {self.name_map[source]}. "
"Permission denied."
)

def outputs_exist(self):
for source in self.name_map:
if not os.path.islink(self.name_map[source]):
# Check if there's a link, NOT whether the source exists.
return False
return True

@classmethod
def get_output_dir(cls, session):
return None

def needs_raw_data(self):
return False


class NiiExporter(SeriesExporter):
"""Export a series to nifti format with datman-style names.
"""
Expand Down
Loading
Loading