Commit 12a1c688 authored by amichaut's avatar amichaut
Browse files

changed filtering usage (parameters saving to be saved)

parent 1c5ba0c0
Pipeline #55020 passed with stages
in 10 seconds
This diff is collapsed.
This diff is collapsed.
......@@ -1525,24 +1525,52 @@ def show_usage():
- plot_vx_vs_vy(data_dir,transform_coord=False,refresh=False,select_ROI=True,set_axis_lim=None)"""
print(usage_message)
def group_consecutives(vals, step=1):
"""
Return list of consecutive lists of numbers from vals (number list).
:param vals:
:param step:
:return:
"""
run = []
result = [run]
expect = None
for v in vals:
if (v == expect) or (expect is None):
run.append(v)
def filter_by_ROI(df, image, filter_all_frames=False, return_ROIs=False):
"""Function used to choose subsets of trajectories which are within ROIs eiher: at a given frame (if filter_all_frames is False), or at all frame (if filter_all_frames is True).
The selection is made by means of a rectangle tool on the image. It can be a 2D, 3D, or 4D image.
t_dim and z_dim give the dimension index of time and z of the nd-image. If None, this dimension doesn't exist """
if image is None:
raise Exception("ERROR: no image provided. Aborting...")
tracks = df.groupby('track')
df_out = pd.DataFrame()
selection = get_coordinates(image)
ROI_list = selection['rectangle']
for i, ROI in enumerate(ROI_list):
xmin, xmax, ymin, ymax = ROI['coord']
frame = ROI['frame']
subdf = pd.DataFrame()
if filter_all_frames:
for fr in df['frame'].unique():
# the subset at the given frame
ind = ((df['frame'] == fr) & (df['x'] >= xmin) & (df['x'] <= xmax) & (df['y'] >= ymin) & (
df['y'] <= ymax))
subdf = pd.concat([subdf, df[ind]])
else:
run = [v]
result.append(run)
expect = v + step
return result
# the subset at the given frame
ind = ((df['frame'] == frame) & (df['x'] >= xmin) & (df['x'] <= xmax) & (df['y'] >= ymin) & (
df['y'] <= ymax))
subdf_frame = df[ind]
# get the subset in the whole dataset
for t in subdf_frame['track'].unique():
track = tracks.get_group(t)
subdf = pd.concat([subdf, track])
# remove empty ROI
if subdf.shape[0] == 0:
print("Warning: ROI #{} doesn't contain any trajectory. ROI skipped...".format(i))
else:
subdf['group'] = i
df_out = pd.concat([df_out, subdf])
if return_ROIs:
return df_out, ROI_list
else:
return df_out
\ No newline at end of file
......@@ -147,9 +147,16 @@ def plot_traj(df, frame, data_dir, groups=None, image={'image_fn': None, 't_dim'
ax = fig.add_subplot(111, projection='3d')
xmin, xmax, ymin, ymax = ax.axis('off')
else:
fig, ax, xmin, ymin, xmax, ymax, no_bkg = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size,
axis_on=show_axis, dpi=plot_config['dpi'])
bkgd = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size,axis_on=show_axis, dpi=plot_config['dpi'])
fig = bkgd['fig']
ax = bkgd['ax']
xmin = bkgd['xmin']
ymin = bkgd['ymin']
xmax = bkgd['xmax']
ymax = bkgd['ymax']
no_bkg = bkgd['no_bkg']
# plotting
groups = df.groupby('frame') if groups is None else groups
group = groups.get_group(frame).reset_index(drop=True)
......@@ -308,9 +315,15 @@ def plot_scalar_field(data_dir, df, data, field, frame, image={'image_fn': None,
if image['image_fn'] is None:
no_bkg = True
fig, ax, xmin, ymin, xmax, ymax, no_bkg = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size, axis_on=show_axis,
dpi=plot_config['dpi'])
bkgd = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size, axis_on=show_axis,dpi=plot_config['dpi'])
fig = bkgd['fig']
ax = bkgd['ax']
xmin = bkgd['xmin']
ymin = bkgd['ymin']
xmax = bkgd['xmax']
ymax = bkgd['ymax']
no_bkg = bkgd['no_bkg']
val_masked = np.ma.array(val, mask=np.isnan(val))
[vmin, vmax] = [val_masked.min(), val_masked.max()] if vlim is None else vlim
......@@ -379,10 +392,17 @@ def plot_vector_field(data_dir, df, data, field, frame, plot_on_field=None, dim=
no_plot_on_field = True
if no_plot_on_field:
fig, ax, xmin, ymin, xmax, ymax, no_bkg = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
bkgd = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size, axis_on=show_axis,
dpi=plot_config['dpi'])
fig = bkgd['fig']
ax = bkgd['ax']
xmin = bkgd['xmin']
ymin = bkgd['ymin']
xmax = bkgd['xmax']
ymax = bkgd['ymax']
no_bkg = bkgd['no_bkg']
# extract data
dimensions = ['x', 'y', 'z'] if dim == 3 else ['x', 'y']
vdata = [field + d for d in dimensions] # eg ['vx','vy'] or ['ax','ay','az']
......@@ -447,9 +467,15 @@ def plot_Voronoi(data_dir, df, frame, data, show_local_area=True,
if image['image_fn'] is None:
no_bkg = True
fig, ax, xmin, ymin, xmax, ymax, no_bkg = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size, axis_on=show_axis,
dpi=plot_config['dpi'])
bkgd = tpr.get_background(image=image, frame=frame, df=df, no_bkg=no_bkg,
image_size=image_size, axis_on=show_axis,dpi=plot_config['dpi'])
fig = bkgd['fig']
ax = bkgd['ax']
xmin = bkgd['xmin']
ymin = bkgd['ymin']
xmax = bkgd['xmax']
ymax = bkgd['ymax']
no_bkg = bkgd['no_bkg']
# plot tesselation
vor = data[frame]['vor']
......@@ -773,7 +799,16 @@ def plot_param_boxplot(data_dir, df, x_param, param, order=None, hue=None, save_
#### PLOT_ALL methods
def view_traj(df, image=None, z_step=1):
"""View trajectories on a napari viewer. Trajectories can be viewed on the original passed by 'image' (a dict containing the path 'image_fn' and the time and z indices 't_dim' and 'z_dim') """
"""
View trajectories on a Napari viewer.
Trajectories can be viewed on the original passed by image
:param df: dataframe of trajectories
:type df: pandas.DataFrame
:param image: image dict returned by prepare.get_image()
:type image: dict
:param z_step:
:type z_step: float or None
"""
with napari.gui_qt():
axis_labels = ['t', 'z', 'x', 'y'] if 'z' in df.columns else ['t', 'x', 'y']
......@@ -788,11 +823,12 @@ def view_traj(df, image=None, z_step=1):
if 'z' in df.columns:
cols = ['frame', 'z', 'y', 'x']
if image['z_dim'] is None:
print(
"WARNING: you have 3D tracking data but your image is not a z-stack, for optimal 3D viewing, use a z-stack")
print("WARNING: you have 3D tracking data but your image is not a z-stack, for optimal 3D "
"viewing, use a z-stack")
viewer.add_image(im, name='image')
else:
viewer.add_image(im, name='image', scale=(1, z_step, 1, 1))
z_step_ = 1 if z_step is None else z_step # in case z_step not given set it to 1
viewer.add_image(im, name='image', scale=(1, z_step_, 1, 1))
else:
cols = ['frame', 'y', 'x']
viewer.add_image(im, name='image')
......
......@@ -863,21 +863,39 @@ def get_traj(track_groups, track, min_frame=None, max_frame=None):
return group.reset_index(drop=True)
def init_filters():
def init_filters(data_dir=None, export_to_config=False):
"""
Initialize database filters
:param data_dir: path to data directory
:type data_dir: str or None
:param export_to_config: to export to config file
:type export_to_config: bool
:return: filters used by select_sub_data()
:rtype: dict
"""
filters = {'xlim': None,
'ylim': None,
'zlim': None,
'min_traj_len': None,
'max_traj_len': None,
'frame_subset': None,
'ROI': None,
'name': '' # custom name to identify the subset
}
# filters for a single subset
subset_dict = {'xlim': None, # x boundaries
'ylim': None, # y boundaries
'zlim': None, # z boundaries
'min_traj_len': None, # minimum trajectory length
'max_traj_len': None, # maximum trajectory length
'frame_subset': None, # frame boundaries
'track_list': None, # a list of trajectories ids
'track_ROI': None, # a dict: {'xlim','ylim','zlim','frame_lim'}
'name': '' # custom name to identify the subset
}
filters = {'subset': 'separately', # analyzed subsets separately or together, options: 'separately','together'
'filters_list': [subset_dict], # list of subsets filters
'subset_order': None, # to give a custom order from plotting subset together
}
if export_to_config:
config_dir = osp.join(data_dir, 'config')
safe_mkdir(config_dir)
fn = osp.join(config_dir, 'filters.csv')
write_dict(filters, fn)
return filters
......@@ -893,17 +911,20 @@ def filter_by_traj_len(df, min_traj_len=1, max_traj_len=None):
:return: filtered dataframe of trajectories
:rtype: pandas.DataFrame
"""
df_ = pd.DataFrame()
if max_traj_len is None: # assign the longest possible track
max_traj_len = df['frame'].max() - df['frame'].min() + 1
min_traj_len = 1 if min_traj_len is None else min_traj_len # assign 1, if not given
tracks = df.groupby('track')
df_list = []
for t in df['track'].unique():
track = tracks.get_group(t)
if track.shape[0] >= min_traj_len and track.shape[0] <= max_traj_len:
df_ = pd.concat([df_, track])
return df_
if (track.shape[0] >= min_traj_len) & (track.shape[0] <= max_traj_len):
df_list.append(track)
out_df = pd.concat(df_list, ignore_index=True)
return out_df
def filter_by_frame_subset(df, frame_subset=None):
......@@ -934,7 +955,7 @@ def filter_by_frame_subset(df, frame_subset=None):
return df_
def region_filter(df, xlim=None, ylim=None, zlim=None):
def filter_by_region(df, xlim=None, ylim=None, zlim=None):
"""
Extract data within a box given by xlim, ylim and zlim in px
:param df: dataframe of trajectories
......@@ -969,7 +990,7 @@ def region_filter(df, xlim=None, ylim=None, zlim=None):
return df_
def get_coordinates(image):
def get_coordinates(image, interactive=True, verbose=True):
"""
Interactive selection of coordinates on an image by hand-drawing using a Napari viewer.
Selection supported: points and rectangle.
......@@ -992,8 +1013,10 @@ def get_coordinates(image):
points_list = []
with napari.gui_qt():
viewer = napari.view_image(im)
print("Draw points or rectangles, then press ENTER and close the image viewer")
if verbose:
print("Draw points or rectangles, then press ENTER and close the image viewer")
# retrieve coodinates on clicking Enter
@viewer.bind_key('Enter')
def get_coord(viewer):
for layer in viewer.layers:
......@@ -1012,14 +1035,20 @@ def get_coordinates(image):
if len(points_list) > 0:
points = points_list[0]
print('You have selected {} point(s) and {} rectangle(s)'.format(points.shape[0], len(rectangle_list)))
finished = input('Is the selection correct? [y]/n: ')
if finished != 'n':
# interactive validation of selection
if verbose:
print('You have selected {} point(s) and {} rectangle(s)'.format(points.shape[0], len(rectangle_list)))
if interactive:
finished = input('Is the selection correct? [y]/n: ')
if finished != 'n':
selecting = False
else:
selecting = False
del viewer
# retreive coordinates
coord_dict = {'points': [], 'rectangle': []}
# get rectangle
# get rectangle coordinates
for rect in rectangle_list:
# if 4D stack
if t_dim is not None and z_dim is not None:
......@@ -1040,8 +1069,9 @@ def get_coordinates(image):
xmin, xmax = [rect[:, 1].min(), rect[:, 1].max()]
ymin, ymax = [rect[:, 0].min(), rect[:, 0].max()]
coord_dict['rectangle'].append({'frame': frame, 'z': z, 'coord': [xmin, xmax, ymin, ymax]})
coord_dict['rectangle'].append({'frame': frame, 'z': z, 'xlim': [xmin, xmax], 'ylim': [ymin, ymax]})
# get points coordinates
for i in range(points.shape[0]):
# if 4D stack
if t_dim is not None and z_dim is not None:
......@@ -1064,53 +1094,59 @@ def get_coordinates(image):
return coord_dict
def filter_by_ROI(df, image, filter_all_frames=False, return_ROIs=False):
"""Function used to choose subsets of trajectories which are within ROIs eiher: at a given frame (if filter_all_frames is False), or at all frame (if filter_all_frames is True).
The selection is made by means of a rectangle tool on the image. It can be a 2D, 3D, or 4D image.
t_dim and z_dim give the dimension index of time and z of the nd-image. If None, this dimension doesn't exist """
def filter_by_traj_id(df, track_list=None):
"""
Filter by trajectory id. Only one id can be given
:param df: dataframe of trajectories
:type df: pandas.DataFrame
:param track_list: list of trajectory ids
:type track_list: list or int or float or None
:return: filtered dataframe of trajectories
:rtype: pandas.DataFrame
"""
if image is None:
raise Exception("ERROR: no image provided. Aborting...")
if track_list is None:
return df
elif type(track_list) is float or type(track_list) is int:
track_list = [track_list]
tracks = df.groupby('track')
df_out = pd.DataFrame()
df_list = []
for track in track_list:
df_list.append(get_traj(tracks, track))
selection = get_coordinates(image)
ROI_list = selection['rectangle']
out_df = pd.concat(df_list, ignore_index=True)
return out_df
for i, ROI in enumerate(ROI_list):
xmin, xmax, ymin, ymax = ROI['coord']
frame = ROI['frame']
subdf = pd.DataFrame()
if filter_all_frames:
for fr in df['frame'].unique():
# the subset at the given frame
ind = ((df['frame'] == fr) & (df['x'] >= xmin) & (df['x'] <= xmax) & (df['y'] >= ymin) & (
df['y'] <= ymax))
subdf = pd.concat([subdf, df[ind]])
else:
# the subset at the given frame
ind = ((df['frame'] == frame) & (df['x'] >= xmin) & (df['x'] <= xmax) & (df['y'] >= ymin) & (
df['y'] <= ymax))
subdf_frame = df[ind]
# get the subset in the whole dataset
for t in subdf_frame['track'].unique():
track = tracks.get_group(t)
subdf = pd.concat([subdf, track])
# remove empty ROI
if subdf.shape[0] == 0:
print("Warning: ROI #{} doesn't contain any trajectory. ROI skipped...".format(i))
else:
subdf['group'] = i
df_out = pd.concat([df_out, subdf])
def select_traj_by_xyzt(df, xlim=None, ylim=None, zlim=None, frame_lim=None):
"""
Get ids of trajectories going through an xyzt box. The spatiotemporal box is defined its boundaries
:param df: dataframe of trajectories
:type df: pandas.DataFrame
:param xlim: x boundaries
:param xlim: list or None
:param ylim: y boundaries
:param ylim: list or None
:param zlim: z boundaries
:param zlim: list or None
:param frame_lim: frame boundaries or unique frame
:param frame_lim: int or list or None
:return: list of trajectories id
:rtype: list
"""
# filter by frame subset
if type(frame_lim) is list:
df = filter_by_frame_subset(df, frame_subset=frame_lim)
elif type(frame_lim) is int or type(frame_lim) is float:
df = df[df['frame'] == frame_lim]
if return_ROIs:
return df_out, ROI_list
else:
return df_out
# filter by region
df = filter_by_region(df, xlim=xlim, ylim=ylim, zlim=zlim)
# get ids
track_list = df['track'].unique()
return track_list
def set_origin(df, image=None, reset_dim=['x', 'y'], lengthscale=1., orig_coord=None):
......@@ -1145,37 +1181,54 @@ def set_origin(df, image=None, reset_dim=['x', 'y'], lengthscale=1., orig_coord=
return df, origin
def select_sub_data(df, image=None, filters=[]):
"""Select with sets of filters given a list (or as a single if only one set). Each set of filter given as a dict {'frame_subset','min_traj_len','filter_by_ROI'}
Each set generates a subet of df. A list of subsets is returned.
def select_sub_data(df, filters=[]):
"""
Select subsets of data according a list of filters. Each element of the list will lead to a subset.
Each subset is filtered by a dict:
{'xlim','ylim','zlim','frame_subset','min_traj_len','max_traj_len','track_list','name'}
:param df: dataframe of trajectories
:type df: pandas.DataFrame
:param filters: list of filters or single set of filters
:type filters: list or dict or None
:return: filtered dataframe of trajectories
:rtype: pandas.DataFrame
"""
df_list = []
ROI_list = []
if len(filters) == 0:
return [df]
# if no filter return input
if filters is None:
return df
elif len(filters) == 0:
return df
if type(filters) is dict: # if only one set of filters
# if only one set of filters
if type(filters) is dict:
filters = [filters]
# perform filtering
df_list = [] # temp list of df
for filt in filters:
df_ = region_filter(df, xlim=filt['xlim'], ylim=filt['ylim'], zlim=filt['zlim'])
# filter
df_ = filter_by_region(df, xlim=filt['xlim'], ylim=filt['ylim'], zlim=filt['zlim'])
df_ = filter_by_frame_subset(df_, frame_subset=filt['frame_subset'])
df_ = filter_by_traj_len(df_, min_traj_len=filt['min_traj_len'], max_traj_len=filt['max_traj_len'])
if filt['ROI'] is not None:
df_, ROIs = filter_by_ROI(df_, image, filter_all_frames=filt['ROI']['filter_all_frames'], return_ROIs=True)
ROI_list.append(ROIs)
else:
ROI_list.append(None)
df_ = filter_by_traj_id(df_, filt['track_list'])
if filt['track_ROI'] is not None:
track_list = select_traj_by_xyzt(df_,xlim=filt['track_ROI']['xlim'], ylim=filt['track_ROI']['ylim'],
zlim=filt['track_ROI']['zlim'], frame_lim=filt['track_ROI']['frame_lim'])
df_ = filter_by_traj_id(df_, track_list)
# subset name
df_['subset'] = filt['name']
df_list.append(df_)
return df_list, ROI_list
out_df = pd.concat(df_list, ignore_index=True)
return out_df
def get_background(image=None, frame=None, df=None, no_bkg=False, image_size=None, orig=None, axis_on=False,
dpi=plot_param['dpi'], figsize=(5, 5)):
"""Get image background or create white backgound if no_bkg. The image can be a time nd stack or a single image."""
"""Get image background or create white background if no_bkg. The image can be a time nd stack or a single image."""
if orig is None:
# orig = 'lower' if image_dir is None else 'upper' #trick to plot for the first time only inverting Yaxis: not very elegant...
orig = 'lower'
......@@ -1233,7 +1286,7 @@ def get_background(image=None, frame=None, df=None, no_bkg=False, image_size=Non
else:
ax.axis('off')
return fig, ax, xmin, ymin, xmax, ymax, no_bkg
return {'fig': fig, 'ax': ax, 'xmin': xmin, 'ymin': ymin, 'xmax': xmax, 'ymax': ymax, 'no_bkg': no_bkg}
def load_config(data_dir, verbose=False):
......@@ -1259,7 +1312,19 @@ def load_config(data_dir, verbose=False):
def get_image(data_dir, filename=None, verbose=False):
""" Get image named 'stack.tif' if exists """
"""
Get a multidimensional image in data directory and analyzes its dimensions.
Convert it to 8bit grayscale if RGB after saving the original image.
:param data_dir: path to data directory
:type data_dir: str
:param filename: custom filaneme if None replace by stack.tif
:type filename: str or None
:param verbose: print the dimension analysis
:type verbose: bool
:return: dict with image path, dimensions (t_dim, z_dim: indices of the t and z dimensions),
image size [height, width] in px
:rtype: dict
"""
filename = osp.join(data_dir, 'stack.tif') if filename is None else filename
......@@ -1279,8 +1344,9 @@ def get_image(data_dir, filename=None, verbose=False):
im = img_as_ubyte(im) # 8bit conversion
tifff.imsave(filename, im)
#
# analyze image dimensions
image_dim = len(im.shape)
z_dim, t_dim = [None, None] # pos
if image_dim == 2:
y_size, x_size = im.shape
z_dim, t_dim = [None, None] # axes of z and t data
......@@ -1290,20 +1356,19 @@ def get_image(data_dir, filename=None, verbose=False):
t_size, y_size, x_size = im.shape
z_dim, t_dim = [None, 0] # axes of z and t data
if verbose:
print("You have loaded a {}D image: ({}x{}) pixels with {} time steps".format(image_dim, x_size, y_size,
t_size))
print("You have loaded a {}D image: "
"({}x{}) pixels with {} time steps".format(image_dim, x_size, y_size, t_size))
elif image_dim == 4:
t_size, z_size, y_size, x_size = im.shape
z_dim, t_dim = [1, 0] # axes of z and t data
if verbose:
print("You have loaded a {}D image: ({}x{}) pixels with {} time steps and {} z slices".format(image_dim,
x_size,
y_size,
t_size,
z_size))
print("You have loaded a {}D image: ({}x{}) pixels with {} time steps "
"and {} z slices".format(image_dim, x_size, y_size, t_size, z_size))
image_dict = {'image_fn': filename, 't_dim': t_dim, 'z_dim': z_dim, 'image_size': im.shape[-2:]}
else:
image_dict = {'image_fn': None, 't_dim': None, 'z_dim': None, 'image_size': None}
return image_dict
......@@ -32,25 +32,26 @@ from track_analyzer import plotting as tpl
from track_analyzer import calculate as tca
def make_traj_config(data_dir=None,export_config=True):
def make_traj_config(data_dir=None, export_config=True):
"""
Generate a list of dictionaries containing the parameters used to run traj_analysis.
"""
traj_config_ = {'run': True, # run plot_traj
'color_code': 'z', # color code: 'z', 'ROI', 'random', 'none'
'cmap' : 'plasma', # colormap to be used if color_code is 'z'
'cmap_lim': None, # pass custom colormap limits (useful for getting coherent boundaries for all frames)
'show_tail': True, # show trajectory tail
'hide_labels': True, # hide trajectory ID
'lab_size': 6, # label size in points if hide_labels is False
'no_bkg': False, # don't show background image if an image is passed
'size_factor': 1., # to multiply the default size of markers and lines
'show_axis': False, # to show the plot axes (by default just image)
'plot3D': False, # plot in 3D
'elevation': None, # 3D paramater
'angle': None # 3D paramater
}
'color_code': 'z', # color code: 'z', 'ROI', 'random', 'none'