Commit 1bcf71dc authored by Dmitry  ERSHOV's avatar Dmitry ERSHOV
Browse files

Add notebook for CSV generation and csv data.

parent 15f2c49e
data/
events.out.tfevents.*
ipynb.checkpoints
.ipynb.checkpoints
.DS_Store
This source diff could not be displayed because it is too large. You can view the blob instead.
%% Cell type:markdown id: tags:
**In case of problems or questions, please first check the list of [Frequently Asked Questions (FAQ)](https://stardist.net/docs/faq.html).**
Please shutdown all other training/prediction notebooks before running this notebook (as those might occupy the GPU memory otherwise).
%% Cell type:code id: tags:
``` python
import sys,os
import pandas as pd
ROOT_DIR = os.path.abspath( '..' )
DATA_DIR = os.path.join( ROOT_DIR, 'data' )
DATA_RAW_DIR = os.path.join( DATA_DIR, '1_raw_movies_and_xmls' )
DATA_GEN_DIR = os.path.join(DATA_DIR, '2_generated' )
IMAGES_DIR = os.path.join(DATA_GEN_DIR, 'images' )
MASKS_DIR = os.path.join(DATA_GEN_DIR, 'masks' )
MODELS_DIR = os.path.join(ROOT_DIR, 'models' )
```
%% Cell type:code id: tags:
``` python
from __future__ import print_function, unicode_literals, absolute_import, division
import sys
import numpy as np
import matplotlib
matplotlib.rcParams["image.interpolation"] = None
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
from glob import glob
from tifffile import imread
from csbdeep.utils import Path, normalize
from csbdeep.io import save_tiff_imagej_compatible
from stardist import random_label_cmap, _draw_polygons, export_imagej_rois
from stardist.models import StarDist2D
np.random.seed(6)
lbl_cmap = random_label_cmap()
```
%% Output
Using TensorFlow backend.
%% Cell type:markdown id: tags:
# Data
- Load only test data; Pay attention to the data that you used for training; dont use it in testing mode.
%% Cell type:code id: tags:
``` python
files = os.listdir(IMAGES_DIR)
tifs = [f for f in files if 'tif' in f]
tifs.sort()
# the naming should be identical; the abs paths to the images (X) and the masks (Y):
imgs = [os.path.join(IMAGES_DIR, t) for t in tifs]
lbls = [os.path.join(MASKS_DIR, t) for t in tifs]
print('Found %d images'%len(imgs))
if False:
# quick look at 10 images
N = 10
_X = list(map(imread, imgs[:N]))
_Y = list(map(imread, lbls[:N]))
n_channel = 1 #if X[0].ndim == 2 else X[0].shape[-1]
print('Test batch: ', len(_X), _X[0].shape, n_channel )
inds_all = np.arange(0, len(imgs))
inds_test = np.arange(0, len(imgs),10)
inds_train = [i for i in inds_all if i not in inds_test]
X_tr = [imgs[i] for i in inds_train]
Y_tr = [lbls[i] for i in inds_train]
X_te = [imgs[i] for i in inds_test]
Y_te = [lbls[i] for i in inds_test]
imgs_te = [imgs[i] for i in inds_test]
lbls_te = [lbls[i] for i in inds_test]
# loading:
X_te = list(map(imread, X_te))
Y_te = list(map(imread, Y_te))
print('Split:\nTRAINING 0.9 (%d pairs)\nTESTING 0.1 (%d pairs)\nTOTAL: %d pairs.'
%(len(X_tr), len(X_te), len(imgs)) )
print('Loaded %d testing images.'%len(X_te))
print('(check pair 0) image shape: ', X_te[0].shape )
print('(check pair 0) labels max: ', Y_te[0].max() )
# stardist related inputs
n_channel = 1 #if X[0].ndim == 2 else X[0].shape[-1]
axis_norm = (0,1)
```
%% Output
Found 3900 images
Split:
TRAINING 0.9 (3510 pairs)
TESTING 0.1 (390 pairs)
TOTAL: 3900 pairs.
Loaded 390 testing images.
(check pair 0) image shape: (512, 512)
(check pair 0) labels max: 639
%% Cell type:markdown id: tags:
# Load trained model
%% Cell type:code id: tags:
``` python
model_name = 'mixAll'
model = StarDist2D(None, name='stardist', basedir=os.path.join(MODELS_DIR, model_name) )
```
%% Output
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.
Loading network weights from 'weights_best.h5'.
Loading thresholds from 'thresholds.json'.
Using default values: prob_thresh=0.469884, nms_thresh=0.4.
%% Cell type:markdown id: tags:
## Prediction
Make sure to normalize the input image beforehand or supply a `normalizer` to the prediction function.
Calling `model.predict_instances` will
- predict object probabilities and star-convex polygon distances (see `model.predict` if you want those)
- perform non-maximum suppression (with overlap threshold `nms_thresh`) for polygons above object probability threshold `prob_thresh`.
- render all remaining polygon instances in a label image
- return the label instances image and also the details (coordinates, etc.) of all remaining polygons
%% Cell type:code id: tags:
``` python
_ind = 16
print(imgs_te[_ind])
img = normalize(X_te[ _ind ], 1,99.8, axis=axis_norm)
labels, details = model.predict_instances(img)
```
%% Output
/master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/images/MICROTUBULE snr 1 density low_00060.tif
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.
%% Cell type:code id: tags:
``` python
plt.figure(figsize=(8,8))
plt.imshow(img if img.ndim==2 else img[...,0], clim=(0,1), cmap='gray')
plt.imshow(labels, cmap=lbl_cmap, alpha=0.5)
# plt.axis('off');
```
%% Output
<matplotlib.image.AxesImage at 0x7fdfed9a06d8>
%% Cell type:markdown id: tags:
## Do all predictions -> CSV
%% Cell type:code id: tags:
``` python
# ID, frame, X, Y, Z, radius, quality
df_img_names = []
df_ids = []
df_frames = []
df_x = []
df_y = []
df_z = []
df_radii = []
df_probs = []
print('Total test images: %d'%len(X_te))
for img_ind, (img_path, img) in enumerate(zip(imgs_te, X_te)):
if img_ind%10==0: print(img_ind, end=',')
img_name = os.path.basename(img_path).replace('.tif','')
fr = int(img_name.split('_')[-1])
# predict, get data
img = normalize(img, 1,99.8, axis=axis_norm)
labels, details = model.predict_instances(img)
coords_rc = details['coord']
points_rc = details['points']
probs = details['prob']
_id = 0
for _coords_rc, _rc, _prob in zip(coords_rc, points_rc, probs ):
# calculate avg radius from all the rays:
dydx = _coords_rc.T - _rc
_ravg = np.sqrt((dydx**2).sum(axis=1)).mean()
# centroid as integer or as float (from rays)
# _y,_x =_rc
_y, _x = _coords_rc.T.mean(axis=0)
df_img_names.append(img_name)
df_frames.append(fr)
df_ids.append(_id)
df_x.append(_x)
df_y.append(_y)
df_z.append(0.)
df_radii.append(_ravg)
df_probs.append(_prob)
_id += 1
# ID, frame, X, Y, Z, radius, quality