Commit 1bcf71dc authored by Dmitry  ERSHOV's avatar Dmitry ERSHOV
Browse files

Add notebook for CSV generation and csv data.

parent 15f2c49e
This source diff could not be displayed because it is too large. You can view the blob instead.
%% Cell type:markdown id: tags:
**In case of problems or questions, please first check the list of [Frequently Asked Questions (FAQ)](**
Please shutdown all other training/prediction notebooks before running this notebook (as those might occupy the GPU memory otherwise).
%% Cell type:code id: tags:
``` python
import sys,os
import pandas as pd
ROOT_DIR = os.path.abspath( '..' )
DATA_DIR = os.path.join( ROOT_DIR, 'data' )
DATA_RAW_DIR = os.path.join( DATA_DIR, '1_raw_movies_and_xmls' )
DATA_GEN_DIR = os.path.join(DATA_DIR, '2_generated' )
IMAGES_DIR = os.path.join(DATA_GEN_DIR, 'images' )
MASKS_DIR = os.path.join(DATA_GEN_DIR, 'masks' )
MODELS_DIR = os.path.join(ROOT_DIR, 'models' )
%% Cell type:code id: tags:
``` python
from __future__ import print_function, unicode_literals, absolute_import, division
import sys
import numpy as np
import matplotlib
matplotlib.rcParams["image.interpolation"] = None
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
from glob import glob
from tifffile import imread
from csbdeep.utils import Path, normalize
from import save_tiff_imagej_compatible
from stardist import random_label_cmap, _draw_polygons, export_imagej_rois
from stardist.models import StarDist2D
lbl_cmap = random_label_cmap()
%% Output
Using TensorFlow backend.
%% Cell type:markdown id: tags:
# Data
- Load only test data; Pay attention to the data that you used for training; dont use it in testing mode.
%% Cell type:code id: tags:
``` python
files = os.listdir(IMAGES_DIR)
tifs = [f for f in files if 'tif' in f]
# the naming should be identical; the abs paths to the images (X) and the masks (Y):
imgs = [os.path.join(IMAGES_DIR, t) for t in tifs]
lbls = [os.path.join(MASKS_DIR, t) for t in tifs]
print('Found %d images'%len(imgs))
if False:
# quick look at 10 images
N = 10
_X = list(map(imread, imgs[:N]))
_Y = list(map(imread, lbls[:N]))
n_channel = 1 #if X[0].ndim == 2 else X[0].shape[-1]
print('Test batch: ', len(_X), _X[0].shape, n_channel )
inds_all = np.arange(0, len(imgs))
inds_test = np.arange(0, len(imgs),10)
inds_train = [i for i in inds_all if i not in inds_test]
X_tr = [imgs[i] for i in inds_train]
Y_tr = [lbls[i] for i in inds_train]
X_te = [imgs[i] for i in inds_test]
Y_te = [lbls[i] for i in inds_test]
imgs_te = [imgs[i] for i in inds_test]
lbls_te = [lbls[i] for i in inds_test]
# loading:
X_te = list(map(imread, X_te))
Y_te = list(map(imread, Y_te))
print('Split:\nTRAINING 0.9 (%d pairs)\nTESTING 0.1 (%d pairs)\nTOTAL: %d pairs.'
%(len(X_tr), len(X_te), len(imgs)) )
print('Loaded %d testing images.'%len(X_te))
print('(check pair 0) image shape: ', X_te[0].shape )
print('(check pair 0) labels max: ', Y_te[0].max() )
# stardist related inputs
n_channel = 1 #if X[0].ndim == 2 else X[0].shape[-1]
axis_norm = (0,1)
%% Output
Found 3900 images
TRAINING 0.9 (3510 pairs)
TESTING 0.1 (390 pairs)
TOTAL: 3900 pairs.
Loaded 390 testing images.
(check pair 0) image shape: (512, 512)
(check pair 0) labels max: 639
%% Cell type:markdown id: tags:
# Load trained model
%% Cell type:code id: tags:
``` python
model_name = 'mixAll'
model = StarDist2D(None, name='stardist', basedir=os.path.join(MODELS_DIR, model_name) )
%% Output
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/ calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/ The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.
Loading network weights from 'weights_best.h5'.
Loading thresholds from 'thresholds.json'.
Using default values: prob_thresh=0.469884, nms_thresh=0.4.
%% Cell type:markdown id: tags:
## Prediction
Make sure to normalize the input image beforehand or supply a `normalizer` to the prediction function.
Calling `model.predict_instances` will
- predict object probabilities and star-convex polygon distances (see `model.predict` if you want those)
- perform non-maximum suppression (with overlap threshold `nms_thresh`) for polygons above object probability threshold `prob_thresh`.
- render all remaining polygon instances in a label image
- return the label instances image and also the details (coordinates, etc.) of all remaining polygons
%% Cell type:code id: tags:
``` python
_ind = 16
img = normalize(X_te[ _ind ], 1,99.8, axis=axis_norm)
labels, details = model.predict_instances(img)
%% Output
/master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/images/MICROTUBULE snr 1 density low_00060.tif
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/ The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.
%% Cell type:code id: tags:
``` python
plt.imshow(img if img.ndim==2 else img[...,0], clim=(0,1), cmap='gray')
plt.imshow(labels, cmap=lbl_cmap, alpha=0.5)
# plt.axis('off');
%% Output
<matplotlib.image.AxesImage at 0x7fdfed9a06d8>
%% Cell type:markdown id: tags:
## Do all predictions -> CSV
%% Cell type:code id: tags:
``` python
# ID, frame, X, Y, Z, radius, quality
df_img_names = []
df_ids = []
df_frames = []
df_x = []
df_y = []
df_z = []
df_radii = []
df_probs = []
print('Total test images: %d'%len(X_te))
for img_ind, (img_path, img) in enumerate(zip(imgs_te, X_te)):
if img_ind%10==0: print(img_ind, end=',')
img_name = os.path.basename(img_path).replace('.tif','')
fr = int(img_name.split('_')[-1])
# predict, get data
img = normalize(img, 1,99.8, axis=axis_norm)
labels, details = model.predict_instances(img)
coords_rc = details['coord']
points_rc = details['points']
probs = details['prob']
_id = 0
for _coords_rc, _rc, _prob in zip(coords_rc, points_rc, probs ):
# calculate avg radius from all the rays:
dydx = _coords_rc.T - _rc
_ravg = np.sqrt((dydx**2).sum(axis=1)).mean()
# centroid as integer or as float (from rays)
# _y,_x =_rc
_y, _x = _coords_rc.T.mean(axis=0)
_id += 1
# ID, frame, X, Y, Z, radius, quality