From 81bc748fb8326c5599e83f41ed39d8ad82289f43 Mon Sep 17 00:00:00 2001 From: swkeemink Date: Thu, 3 Jun 2021 19:06:43 +0200 Subject: [PATCH 01/33] API: datahandler is now a class This involves quite a lot of changes. This is classed as an API change as anyone using custom dataloaders in our previous way will either have to stay with an older version of FISSA, or migrate their datahandlers to this new method. I will describe this in detail in the tutorial notebook that is to follow in another commit. --- fissa/core.py | 25 ++-- fissa/datahandler.py | 216 ++++++++++++++--------------- fissa/datahandler_framebyframe.py | 218 +++++++++++++++--------------- 3 files changed, 236 insertions(+), 223 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 39110d23..88e3f765 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -18,7 +18,6 @@ import numpy as np from scipy.io import savemat -from . import datahandler from . import deltaf from . import neuropil as npil from . import roitools @@ -48,6 +47,7 @@ def extract_func(inputs): rois = inputs[1] nNpil = inputs[2] expansion = inputs[3] + datahandler = inputs[4] # get data as arrays and rois as masks curdata = datahandler.image2array(image) @@ -107,6 +107,7 @@ def separate_func(inputs): X = inputs[0] alpha = inputs[1] method = inputs[2] + Xsep, Xmatch, Xmixmat, convergence = npil.separate( X, method, maxiter=20000, tol=1e-4, maxtries=1, alpha=alpha ) @@ -186,9 +187,11 @@ def __init__(self, images, rois, folder=None, nRegions=4, option reduces the memory load, and may be necessary for very large inputs. Default is ``False``. datahandler_custom : object, optional - A custom datahandler for handling ROIs and calcium data can + A custom datahandler class for handling ROIs and calcium data can be given here. See datahandler.py (the default handler) for - an example. + an example. Easiest way to go about it usually to inheret the standard + datahandler class. Note: will overwrite the datahandler used for + lowmemory_mode. """ if isinstance(images, basestring): @@ -209,11 +212,15 @@ def __init__(self, images, rois, folder=None, nRegions=4, self.rois *= len(self.images) else: raise ValueError('rois should either be string or list') - global datahandler - if lowmemory_mode: - from . import datahandler_framebyframe as datahandler - if datahandler_custom is not None: - datahandler = datahandler_custom + + if datahandler_custom is None: + if lowmemory_mode: + from .datahandler_framebyframe import DataHandler + else: + from .datahandler import DataHandler + self.datahandler = DataHandler() + else: + self.datahandler = datahandler_custom # define class variables self.folder = folder @@ -294,7 +301,7 @@ def separation_prep(self, redo=False): inputs = [0] * self.nTrials for trial in range(self.nTrials): inputs[trial] = [self.images[trial], self.rois[trial], - self.nRegions, self.expansion] + self.nRegions, self.expansion, self.datahandler] # Check whether we should use multiprocessing use_multiprocessing = ( diff --git a/fissa/datahandler.py b/fissa/datahandler.py index 622fb8f3..2def7b74 100755 --- a/fissa/datahandler.py +++ b/fissa/datahandler.py @@ -19,112 +19,116 @@ from . import roitools +class DataHandler(): + """Contains all data interaction functions.""" + def __init__(self): + pass + + def image2array(self, image): + """Loads a TIFF image from disk. + + Parameters + ---------- + image : str or array_like + Either a path to a TIFF file, or array_like data. + + Returns + ------- + numpy.ndarray + A 3D array containing the data, with dimensions corresponding to + `(frames, y_coordinate, x_coordinate)`. + + """ + if isinstance(image, basestring): + return tifffile.imread(image) + + + return np.array(image) + + def getmean(self, data): + """Determine the mean image across all frames. + + Parameters + ---------- + data : array_like + Data array as made by image2array. Should be shaped `(frames, y, x)`. + + Returns + ------- + numpy.ndarray + y by x array for the mean values + + """ + return data.mean(axis=0) + + + def rois2masks(self, rois, data): + """Take the object `rois` and returns it as a list of binary masks. + + Parameters + ---------- + rois : string or list of array_like + Either a string with imagej roi zip location, list of arrays encoding + polygons, or list of binary arrays representing masks + data : array + Data array as made by image2array. Must be shaped `(frames, y, x)`. + + Returns + ------- + list + List of binary arrays (i.e. masks) + + """ + # get the image shape + shape = data.shape[1:] + + # if it's a list of strings + if isinstance(rois, basestring): + rois = roitools.readrois(rois) + + if not isinstance(rois, collections.Sequence): + raise TypeError( + 'Wrong ROIs input format: expected a list or sequence, but got' + ' a {}'.format(rois.__class__) + ) + + # if it's a something by 2 array (or vice versa), assume polygons + if np.shape(rois[0])[1] == 2 or np.shape(rois[0])[0] == 2: + return roitools.getmasks(rois, shape) + # if it's a list of bigger arrays, assume masks + elif np.shape(rois[0]) == shape: + return rois + + raise ValueError('Wrong ROIs input format: unfamiliar shape.') + + + def extracttraces(self, data, masks): + """Extracts a temporal trace for each spatial mask. + + Parameters + ---------- + data : array_like + Data array as made by image2array. Should be shaped + `(frames, y, x)`. + masks : list of array_like + List of binary arrays. + + Returns + ------- + numpy.ndarray + Trace for each mask. Shaped `(len(masks), n_frames)`. + + """ + # get the number rois and frames + nrois = len(masks) + nframes = data.shape[0] -def image2array(image): - """Loads a TIFF image from disk. + # predefine output data + out = np.zeros((nrois, nframes)) - Parameters - ---------- - image : str or array_like - Either a path to a TIFF file, or array_like data. + # loop over masks + for i in range(nrois): # for masks + # get mean data from mask + out[i, :] = data[:, masks[i]].mean(axis=1) - Returns - ------- - numpy.ndarray - A 3D array containing the data, with dimensions corresponding to - `(frames, y_coordinate, x_coordinate)`. - - """ - if isinstance(image, basestring): - return tifffile.imread(image) - - return np.array(image) - - -def getmean(data): - """Determine the mean image across all frames. - - Parameters - ---------- - data : array_like - Data array as made by image2array. Should be shaped `(frames, y, x)`. - - Returns - ------- - numpy.ndarray - y by x array for the mean values - - """ - return data.mean(axis=0) - - -def rois2masks(rois, data): - """Take the object `rois` and returns it as a list of binary masks. - - Parameters - ---------- - rois : string or list of array_like - Either a string with imagej roi zip location, list of arrays encoding - polygons, or list of binary arrays representing masks - data : array - Data array as made by image2array. Must be shaped `(frames, y, x)`. - - Returns - ------- - list - List of binary arrays (i.e. masks) - - """ - # get the image shape - shape = data.shape[1:] - - # if it's a list of strings - if isinstance(rois, basestring): - rois = roitools.readrois(rois) - - if not isinstance(rois, collections.Sequence): - raise TypeError( - 'Wrong ROIs input format: expected a list or sequence, but got' - ' a {}'.format(rois.__class__) - ) - - # if it's a something by 2 array (or vice versa), assume polygons - if np.shape(rois[0])[1] == 2 or np.shape(rois[0])[0] == 2: - return roitools.getmasks(rois, shape) - # if it's a list of bigger arrays, assume masks - elif np.shape(rois[0]) == shape: - return rois - - raise ValueError('Wrong ROIs input format: unfamiliar shape.') - - -def extracttraces(data, masks): - """Extracts a temporal trace for each spatial mask. - - Parameters - ---------- - data : array_like - Data array as made by image2array. Should be shaped - `(frames, y, x)`. - masks : list of array_like - List of binary arrays. - - Returns - ------- - numpy.ndarray - Trace for each mask. Shaped `(len(masks), n_frames)`. - - """ - # get the number rois and frames - nrois = len(masks) - nframes = data.shape[0] - - # predefine output data - out = np.zeros((nrois, nframes)) - - # loop over masks - for i in range(nrois): # for masks - # get mean data from mask - out[i, :] = data[:, masks[i]].mean(axis=1) - - return out + return out diff --git a/fissa/datahandler_framebyframe.py b/fissa/datahandler_framebyframe.py index 50ae04d7..a729018a 100644 --- a/fissa/datahandler_framebyframe.py +++ b/fissa/datahandler_framebyframe.py @@ -18,136 +18,138 @@ from PIL import Image, ImageSequence from . import roitools +from . import datahandler -def image2array(image): - """Open a given image file as a PIL.Image instance. +class DataHandler(datahandler.DataHandler): + def image2array(self, image): + """Open a given image file as a PIL.Image instance. - Parameters - ---------- - image : str or file - A filename (string) of a TIFF image file, a pathlib.Path object, - or a file object. + Parameters + ---------- + image : str or file + A filename (string) of a TIFF image file, a pathlib.Path object, + or a file object. + + Returns + ------- + PIL.Image + Handle from which frames can be loaded. - Returns - ------- - PIL.Image - Handle from which frames can be loaded. + """ + return Image.open(image) - """ - return Image.open(image) + def getmean(self, data): + """Determine the mean image across all frames. + + Parameters + ---------- + data : PIL.Image + An open PIL.Image handle to a multi-frame TIFF image. -def getmean(data): - """Determine the mean image across all frames. + Returns + ------- + numpy.ndarray + y-by-x array for the mean values. - Parameters - ---------- - data : PIL.Image - An open PIL.Image handle to a multi-frame TIFF image. + """ + # We don't load the entire image into memory at once, because + # it is likely to be rather large. + # Initialise holding array with zeros + avg = np.zeros(data.size[::-1]) - Returns - ------- - numpy.ndarray - y-by-x array for the mean values. + # Make sure we seek to the first frame before iterating. This is + # because the Iterator outputs the value for the current frame for + # `img` first, due to a bug in Pillow<=3.1. + data.seek(0) - """ - # We don't load the entire image into memory at once, because - # it is likely to be rather large. - # Initialise holding array with zeros - avg = np.zeros(data.size[::-1]) + # Loop over all frames and sum the pixel intensities together + for frame in ImageSequence.Iterator(data): + avg += np.asarray(frame) - # Make sure we seek to the first frame before iterating. This is - # because the Iterator outputs the value for the current frame for - # `img` first, due to a bug in Pillow<=3.1. - data.seek(0) + # Divide by number of frames to find the average + avg /= data.n_frames + return avg - # Loop over all frames and sum the pixel intensities together - for frame in ImageSequence.Iterator(data): - avg += np.asarray(frame) - # Divide by number of frames to find the average - avg /= data.n_frames - return avg + def rois2masks(self, rois, data): + """Take the object 'rois' and returns it as a list of binary masks. + Parameters + ---------- + rois : str or list of array_like + Either a string with imagej roi zip location, list of arrays encoding + polygons, or list of binary arrays representing masks + data : PIL.Image + An open PIL.Image handle to a multi-frame TIFF image. -def rois2masks(rois, data): - """Take the object 'rois' and returns it as a list of binary masks. + Returns + ------- + list + List of binary arrays (i.e. masks). - Parameters - ---------- - rois : str or list of array_like - Either a string with imagej roi zip location, list of arrays encoding - polygons, or list of binary arrays representing masks - data : PIL.Image - An open PIL.Image handle to a multi-frame TIFF image. + """ + # get the image shape + shape = data.size[::-1] - Returns - ------- - list - List of binary arrays (i.e. masks). + # If rois is string, we first need to read the contents of the file + if isinstance(rois, basestring): + rois = roitools.readrois(rois) - """ - # get the image shape - shape = data.size[::-1] + if not isinstance(rois, collections.Sequence): + raise TypeError( + 'Wrong ROIs input format: expected a list or sequence, but got' + ' a {}'.format(rois.__class__) + ) - # If rois is string, we first need to read the contents of the file - if isinstance(rois, basestring): - rois = roitools.readrois(rois) + # if it's a something by 2 array (or vice versa), assume polygons + if np.shape(rois[0])[1] == 2 or np.shape(rois[0])[0] == 2: + return roitools.getmasks(rois, shape) + # if it's a list of bigger arrays, assume masks + elif np.shape(rois[0]) == shape: + return rois - if not isinstance(rois, collections.Sequence): - raise TypeError( - 'Wrong ROIs input format: expected a list or sequence, but got' - ' a {}'.format(rois.__class__) - ) + raise ValueError('Wrong ROIs input format: unfamiliar shape.') - # if it's a something by 2 array (or vice versa), assume polygons - if np.shape(rois[0])[1] == 2 or np.shape(rois[0])[0] == 2: - return roitools.getmasks(rois, shape) - # if it's a list of bigger arrays, assume masks - elif np.shape(rois[0]) == shape: - return rois - raise ValueError('Wrong ROIs input format: unfamiliar shape.') + def extracttraces(self, data, masks): + """Get the traces for each mask in masks from data. + Parameters + ---------- + data : PIL.Image + An open PIL.Image handle to a multi-frame TIFF image. + masks : list of array_like + List of binary arrays. + + Returns + ------- + numpy.ndarray + Trace for each mask. Shaped `(len(masks), n_frames)`. + + """ + # get the number rois + nrois = len(masks) + + # get number of frames, and start at zeros + data.seek(0) + nframes = data.n_frames -def extracttraces(data, masks): - """Get the traces for each mask in masks from data. + # predefine array with the data + out = np.zeros((nrois, nframes)) - Parameters - ---------- - data : PIL.Image - An open PIL.Image handle to a multi-frame TIFF image. - masks : list of array_like - List of binary arrays. - - Returns - ------- - numpy.ndarray - Trace for each mask. Shaped `(len(masks), n_frames)`. - - """ - # get the number rois - nrois = len(masks) - - # get number of frames, and start at zeros - data.seek(0) - nframes = data.n_frames - - # predefine array with the data - out = np.zeros((nrois, nframes)) - - # for each frame, get the data - for f in range(nframes): - # set frame - data.seek(f) - - # make numpy array - curframe = np.asarray(data) - - # loop over masks - for i in range(nrois): - # get mean data from mask - out[i, f] = np.mean(curframe[masks[i]]) - - return out + # for each frame, get the data + for f in range(nframes): + # set frame + data.seek(f) + + # make numpy array + curframe = np.asarray(data) + + # loop over masks + for i in range(nrois): + # get mean data from mask + out[i, f] = np.mean(curframe[masks[i]]) + + return out From 715405da220cdf51e4178c97735e02e1f8106b08 Mon Sep 17 00:00:00 2001 From: swkeemink Date: Thu, 3 Jun 2021 19:07:32 +0200 Subject: [PATCH 02/33] JNB: updated the tutorial notebook for the new datahandler method --- examples/Basic usage.html | 27664 ++++++++++++++++++----------------- examples/Basic usage.ipynb | 1822 +-- 2 files changed, 14579 insertions(+), 14907 deletions(-) diff --git a/examples/Basic usage.html b/examples/Basic usage.html index 6db19429..339425bd 100644 --- a/examples/Basic usage.html +++ b/examples/Basic usage.html @@ -1,13147 +1,14326 @@ + -Basic usage +Basic usage - - + + + + - +.jp-KeySelector select.jp-mod-styled { + font-size: var(--jp-ui-font-size1); + color: var(--jp-ui-font-color0); + border: var(--jp-border-width) solid var(--jp-border-color1); +} + +.jp-KeySelector label, +.jp-MetadataEditorTool label { + line-height: 1.4; +} + +/*----------------------------------------------------------------------------- +| Presentation Mode (.jp-mod-presentationMode) +|----------------------------------------------------------------------------*/ + +.jp-mod-presentationMode .jp-Notebook { + --jp-content-font-size1: var(--jp-content-presentation-font-size1); + --jp-code-font-size: var(--jp-code-presentation-font-size); +} + +.jp-mod-presentationMode .jp-Notebook .jp-Cell .jp-InputPrompt, +.jp-mod-presentationMode .jp-Notebook .jp-Cell .jp-OutputPrompt { + flex: 0 0 110px; +} + +/*----------------------------------------------------------------------------- +| Copyright (c) Jupyter Development Team. +| Distributed under the terms of the Modified BSD License. +|----------------------------------------------------------------------------*/ + +/* This file was auto-generated by ensurePackage() in @jupyterlab/buildutils */ + +/*----------------------------------------------------------------------------- +| Copyright (c) Jupyter Development Team. +| Distributed under the terms of the Modified BSD License. +|----------------------------------------------------------------------------*/ + + + + - - - + - + - -
-
+ -
-
-
+ -
-
-
-
+ -
-
-
-
In [1]:
-
-
+
-
-
-
+ -
-
-
-
In [2]:
-
-
+
-
-
+