Commit 2581b1ac authored by Dmitry  ERSHOV's avatar Dmitry ERSHOV
Browse files

First commit

parents
data/
events.out.tfevents.*
ipynb.chckpoints
.DS_Store
Implementation of Stardist for Trackmate
- Generate pairs image+labels out of the raw tif stacks (100 frames) and xml files (coordinates)
- Use these images to train Stardist models
- Detect particles with the model of choice.
{"n_dim": 2, "axes": "YXC", "n_channel_in": 1, "n_channel_out": 33, "train_checkpoint": "weights_best.h5", "train_checkpoint_last": "weights_last.h5", "train_checkpoint_epoch": "weights_now.h5", "n_rays": 32, "grid": [2, 2], "backbone": "unet", "unet_n_depth": 3, "unet_kernel_size": [3, 3], "unet_n_filter_base": 32, "unet_n_conv_per_depth": 2, "unet_pool": [2, 2], "unet_activation": "relu", "unet_last_activation": "relu", "unet_batch_norm": false, "unet_dropout": 0.0, "unet_prefix": "", "net_conv_after_unet": 128, "net_input_shape": [null, null, 1], "net_mask_shape": [null, null, 1], "train_shape_completion": false, "train_completion_crop": 32, "train_patch_size": [256, 256], "train_background_reg": 0.0001, "train_foreground_only": 0.9, "train_dist_loss": "mae", "train_loss_weights": [1, 0.2], "train_epochs": 100, "train_steps_per_epoch": 100, "train_learning_rate": 0.0003, "train_batch_size": 4, "train_n_val_patches": null, "train_tensorboard": true, "train_reduce_lr": {"factor": 0.5, "patience": 40, "min_delta": 0}, "use_gpu": true}
\ No newline at end of file
{"n_dim": 2, "axes": "YXC", "n_channel_in": 1, "n_channel_out": 33, "train_checkpoint": "weights_best.h5", "train_checkpoint_last": "weights_last.h5", "train_checkpoint_epoch": "weights_now.h5", "n_rays": 32, "grid": [2, 2], "backbone": "unet", "unet_n_depth": 3, "unet_kernel_size": [3, 3], "unet_n_filter_base": 32, "unet_n_conv_per_depth": 2, "unet_pool": [2, 2], "unet_activation": "relu", "unet_last_activation": "relu", "unet_batch_norm": false, "unet_dropout": 0.0, "unet_prefix": "", "net_conv_after_unet": 128, "net_input_shape": [null, null, 1], "net_mask_shape": [null, null, 1], "train_shape_completion": false, "train_completion_crop": 32, "train_patch_size": [256, 256], "train_background_reg": 0.0001, "train_foreground_only": 0.9, "train_dist_loss": "mae", "train_loss_weights": [1, 0.2], "train_epochs": 200, "train_steps_per_epoch": 100, "train_learning_rate": 0.0003, "train_batch_size": 4, "train_n_val_patches": null, "train_tensorboard": true, "train_reduce_lr": {"factor": 0.5, "patience": 40, "min_delta": 0}, "use_gpu": true}
\ No newline at end of file
{"prob": 0.46164458990097046, "nms": 0.4}
\ No newline at end of file
{"n_dim": 2, "axes": "YXC", "n_channel_in": 1, "n_channel_out": 33, "train_checkpoint": "weights_best.h5", "train_checkpoint_last": "weights_last.h5", "train_checkpoint_epoch": "weights_now.h5", "n_rays": 32, "grid": [2, 2], "backbone": "unet", "unet_n_depth": 3, "unet_kernel_size": [3, 3], "unet_n_filter_base": 32, "unet_n_conv_per_depth": 2, "unet_pool": [2, 2], "unet_activation": "relu", "unet_last_activation": "relu", "unet_batch_norm": false, "unet_dropout": 0.0, "unet_prefix": "", "net_conv_after_unet": 128, "net_input_shape": [null, null, 1], "net_mask_shape": [null, null, 1], "train_shape_completion": false, "train_completion_crop": 32, "train_patch_size": [256, 256], "train_background_reg": 0.0001, "train_foreground_only": 0.9, "train_dist_loss": "mae", "train_loss_weights": [1, 0.2], "train_epochs": 200, "train_steps_per_epoch": 100, "train_learning_rate": 0.0003, "train_batch_size": 4, "train_n_val_patches": null, "train_tensorboard": true, "train_reduce_lr": {"factor": 0.5, "patience": 40, "min_delta": 0}, "use_gpu": true}
\ No newline at end of file
{"prob": 0.46988391876220703, "nms": 0.4}
\ No newline at end of file
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### This notebook is to generate training data from the original tifs and xmls"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Made dir: /master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/images\n",
"Made dir: /master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/masks\n"
]
}
],
"source": [
"import os, sys\n",
"import xml.etree.ElementTree as ET\n",
"import pandas as pd\n",
"import tifffile\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from skimage.morphology import dilation, selem\n",
"\n",
"ROOT_DIR = os.path.abspath( '..' )\n",
"\n",
"DATA_DIR = os.path.join( ROOT_DIR, 'data' )\n",
"DATA_RAW_DIR = os.path.join( DATA_DIR, '1_raw_movies_and_xmls' )\n",
"DATA_GEN_DIR = os.path.join(DATA_DIR, '2_generated' )\n",
"IMAGES_DIR = os.path.join(DATA_GEN_DIR, 'images' )\n",
"MASKS_DIR = os.path.join(DATA_GEN_DIR, 'masks' )\n",
"\n",
"\n",
"\n",
"# TR_IMAGES_DIR = os.path.join(TRAINING_DIR, 'images' )\n",
"# TR_MASKS_DIR = os.path.join(TRAINING_DIR, 'masks' )\n",
"\n",
"# TEST_DIR = os.path.join(DATA_DIR, 'test' )\n",
"# TE_IMAGES_DIR = os.path.join(TEST_DIR, 'images' )\n",
"# TE_MASKS_DIR = os.path.join(TEST_DIR, 'masks' )\n",
"\n",
"\n",
"# for f in [TRAINING_DIR, TEST_DIR, TR_IMAGES_DIR, TR_MASKS_DIR, TE_IMAGES_DIR, TE_MASKS_DIR]:\n",
"for f in [IMAGES_DIR, MASKS_DIR]:\n",
" if not os.path.exists(f): \n",
" os.makedirs(f)\n",
" print('Made dir: %s'%f)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def read_xml_as_pands( xml_abs_path ):\n",
" \n",
" tree = ET.parse( xml_abs_path )\n",
" root = tree.getroot()\n",
" particles = root[0]\n",
"\n",
" # the fastes way is to first collect all the data and then feed it to pandas.\n",
" # if you create pandas in each loop step, it will be way longer\n",
" ii, tt, xx, yy, zz = [], [],[],[],[]\n",
"\n",
" for pi, detections in enumerate(particles): #over particles\n",
" for d in detections: # over each p's detectoins\n",
" t,x,y,z = d.attrib['t'], d.attrib['x'], d.attrib['y'], d.attrib['z']\n",
" ii.append(pi)\n",
" tt.append(t)\n",
" xx.append(x)\n",
" yy.append(y)\n",
" zz.append(z) \n",
"\n",
" # feed the collected data to pandas\n",
" df = pd.DataFrame(data={'pid':ii, 'fr':tt, 'x':xx, 'y':yy, 'z':zz} )\n",
" for n in ['x','y','z']:\n",
" df[n] = pd.to_numeric(df[n], downcast='float')\n",
"\n",
" # it is all strings now; convert to numeric\n",
" df['fr'] = pd.to_numeric(df['fr'], downcast='integer')\n",
" df['pid'] = pd.to_numeric(df['pid'], downcast='integer')\n",
"\n",
" return df\n",
"\n",
"\n",
"\n",
"def df_to_lbl( img, df, radius ):\n",
" # df = ['pid', 'fr', 'x', 'y', 'z']\n",
" \n",
" id_max = df['pid'].max()\n",
" assert id_max < 2**16-1, \"Label image is coded to be unsigned 16bit; you have %d particles.\"%id_max\n",
" \n",
" # dilate with:\n",
" se = selem.disk(radius)\n",
" \n",
" frames_n, rows, cols = img.shape\n",
" img_lbl = np.zeros_like(img).astype(np.uint16)\n",
"\n",
" for _fr in range(frames_n):\n",
" _df = df[df['fr']==_fr] # df for cur frame\n",
" for i,_d in _df.iterrows(): # df for cur particle in this frame\n",
" pid, c, r = int(_d['pid']), int(np.round(_d['x'])), int(np.round(_d['y']))\n",
" img_lbl[_fr, r, c] = pid # mark a pixel with particle id\n",
" img_lbl[_fr] = dilation( img_lbl[_fr], selem=se)\n",
" \n",
" return img_lbl\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MICROTUBULE snr 1 density high.xml \t MICROTUBULE snr 1 density high.tif\n",
"MICROTUBULE snr 1 density low.xml \t MICROTUBULE snr 1 density low.tif\n",
"MICROTUBULE snr 1 density mid.xml \t MICROTUBULE snr 1 density mid.tif\n",
"MICROTUBULE snr 2 density high.xml \t MICROTUBULE snr 2 density high.tif\n",
"MICROTUBULE snr 2 density low.xml \t MICROTUBULE snr 2 density low.tif\n",
"MICROTUBULE snr 2 density mid.xml \t MICROTUBULE snr 2 density mid.tif\n",
"MICROTUBULE snr 4 density high.xml \t MICROTUBULE snr 4 density high.tif\n",
"MICROTUBULE snr 4 density low.xml \t MICROTUBULE snr 4 density low.tif\n",
"MICROTUBULE snr 4 density mid.xml \t MICROTUBULE snr 4 density mid.tif\n",
"MICROTUBULE snr 7 density high.xml \t MICROTUBULE snr 7 density high.tif\n",
"MICROTUBULE snr 7 density low.xml \t MICROTUBULE snr 7 density low.tif\n",
"MICROTUBULE snr 7 density mid.xml \t MICROTUBULE snr 7 density mid.tif\n",
"RECEPTOR snr 1 density high.xml \t RECEPTOR snr 1 density high.tif\n",
"RECEPTOR snr 1 density low.xml \t RECEPTOR snr 1 density low.tif\n",
"RECEPTOR snr 1 density mid.xml \t RECEPTOR snr 1 density mid.tif\n",
"RECEPTOR snr 2 density high.xml \t RECEPTOR snr 2 density high.tif\n",
"RECEPTOR snr 2 density low.xml \t RECEPTOR snr 2 density low.tif\n",
"RECEPTOR snr 2 density mid.xml \t RECEPTOR snr 2 density mid.tif\n",
"RECEPTOR snr 3 density high.xml \t RECEPTOR snr 3 density high.tif\n",
"RECEPTOR snr 3 density low.xml \t RECEPTOR snr 3 density low.tif\n",
"RECEPTOR snr 3 density mid.xml \t RECEPTOR snr 3 density mid.tif\n",
"RECEPTOR snr 4 density high.xml \t RECEPTOR snr 4 density high.tif\n",
"RECEPTOR snr 4 density low.xml \t RECEPTOR snr 4 density low.tif\n",
"RECEPTOR snr 4 density mid.xml \t RECEPTOR snr 4 density mid.tif\n",
"RECEPTOR snr 7 density high.xml \t RECEPTOR snr 7 density high.tif\n",
"RECEPTOR snr 7 density low.xml \t RECEPTOR snr 7 density low.tif\n",
"RECEPTOR snr 7 density mid.xml \t RECEPTOR snr 7 density mid.tif\n",
"VESICLE snr 1 density high.xml \t VESICLE snr 1 density high.tif\n",
"VESICLE snr 1 density low.xml \t VESICLE snr 1 density low.tif\n",
"VESICLE snr 1 density mid.xml \t VESICLE snr 1 density mid.tif\n",
"VESICLE snr 2 density high.xml \t VESICLE snr 2 density high.tif\n",
"VESICLE snr 2 density low.xml \t VESICLE snr 2 density low.tif\n",
"VESICLE snr 2 density mid.xml \t VESICLE snr 2 density mid.tif\n",
"VESICLE snr 4 density high.xml \t VESICLE snr 4 density high.tif\n",
"VESICLE snr 4 density low.xml \t VESICLE snr 4 density low.tif\n",
"VESICLE snr 4 density mid.xml \t VESICLE snr 4 density mid.tif\n",
"VESICLE snr 7 density high.xml \t VESICLE snr 7 density high.tif\n",
"VESICLE snr 7 density low.xml \t VESICLE snr 7 density low.tif\n",
"VESICLE snr 7 density mid.xml \t VESICLE snr 7 density mid.tif\n"
]
}
],
"source": [
"files = os.listdir(DATA_RAW_DIR)\n",
"tifs = [f for f in files if f.endswith('.tif')]\n",
"xmls = [f for f in files if f.endswith('.xml')]\n",
"\n",
"# sort the tifs according to the xmls file order:\n",
"xmls.sort()\n",
"tifs_sorted = []\n",
"\n",
"for xmlname in xmls:\n",
" tifname = xmlname.replace('xml','tif')\n",
" if tifname in tifs:\n",
" tifs_sorted.append( tifname )\n",
"tifs = tifs_sorted\n",
"\n",
"for x,t in zip( xmls, tifs):\n",
" print('%s \\t %s'%(x,t))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MICROTUBULE snr 1 density high.... DONE.\n",
"MICROTUBULE snr 1 density low.... DONE.\n",
"MICROTUBULE snr 1 density mid.... DONE.\n",
"MICROTUBULE snr 2 density high.... DONE.\n",
"MICROTUBULE snr 2 density low.... DONE.\n",
"MICROTUBULE snr 2 density mid.... DONE.\n",
"MICROTUBULE snr 4 density high.... DONE.\n",
"MICROTUBULE snr 4 density low.... DONE.\n",
"MICROTUBULE snr 4 density mid.... DONE.\n",
"MICROTUBULE snr 7 density high.... DONE.\n",
"MICROTUBULE snr 7 density low.... DONE.\n",
"MICROTUBULE snr 7 density mid.... DONE.\n",
"RECEPTOR snr 1 density high.... DONE.\n",
"RECEPTOR snr 1 density low.... DONE.\n",
"RECEPTOR snr 1 density mid.... DONE.\n",
"RECEPTOR snr 2 density high.... DONE.\n",
"RECEPTOR snr 2 density low.... DONE.\n",
"RECEPTOR snr 2 density mid.... DONE.\n",
"RECEPTOR snr 3 density high.... DONE.\n",
"RECEPTOR snr 3 density low.... DONE.\n",
"RECEPTOR snr 3 density mid.... DONE.\n",
"RECEPTOR snr 4 density high.... DONE.\n",
"RECEPTOR snr 4 density low.... DONE.\n",
"RECEPTOR snr 4 density mid.... DONE.\n",
"RECEPTOR snr 7 density high.... DONE.\n",
"RECEPTOR snr 7 density low.... DONE.\n",
"RECEPTOR snr 7 density mid.... DONE.\n",
"VESICLE snr 1 density high.... DONE.\n",
"VESICLE snr 1 density low.... DONE.\n",
"VESICLE snr 1 density mid.... DONE.\n",
"VESICLE snr 2 density high.... DONE.\n",
"VESICLE snr 2 density low.... DONE.\n",
"VESICLE snr 2 density mid.... DONE.\n",
"VESICLE snr 4 density high.... DONE.\n",
"VESICLE snr 4 density low.... DONE.\n",
"VESICLE snr 4 density mid.... DONE.\n",
"VESICLE snr 7 density high.... DONE.\n",
"VESICLE snr 7 density low.... DONE.\n",
"VESICLE snr 7 density mid.... DONE.\n",
"Done all.\n"
]
}
],
"source": [
"R = 3\n",
"# frames_for_test_data = np.arange(0,99,10) # 10 out of 100\n",
"\n",
"for xml_file, tif_file in zip(xmls, tifs):\n",
"\n",
" base_name = os.path.splitext(xml_file)[0]\n",
" print(base_name, end='....')\n",
"\n",
" # read\n",
" df = read_xml_as_pands( os.path.join(DATA_RAW_DIR, xml_file) )\n",
" img = tifffile.imread( os.path.join(DATA_RAW_DIR, tif_file))\n",
"\n",
" img_lbl = df_to_lbl( img, df, radius=R )\n",
" \n",
" # for each frame:\n",
" for _fr, (_im, _lbl) in enumerate(zip(img, img_lbl)):\n",
" _fname = '%s_%05d.tif'%(base_name, _fr)\n",
" \n",
" img_abs_path = os.path.join( IMAGES_DIR, _fname )\n",
" lbl_abs_path = os.path.join( MASKS_DIR, _fname )\n",
" \n",
" # print( ' ', img_abs_path )\n",
" # print( ' ', lbl_abs_path )\n",
" tifffile.imwrite(img_abs_path, data=_im )\n",
" tifffile.imwrite(lbl_abs_path, data=_lbl )\n",
" \n",
" print(' DONE.')\n",
" \n",
"print('Done all.')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(['VESICLE snr 7 density mid.xml'], ['VESICLE snr 7 density mid.tif'])"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#VESICLE snr 7 density mid\n",
"# target = 'VESICLE snr 7 density mid'\n",
"# xmls_chosen = [x for x in xmls if target in x]\n",
"# tifs_chosen = [x for x in tifs if target in x]\n",
"# xmls_chosen, tifs_chosen"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"# xmls_chosen = xmls\n",
"# tifs_chosen = tifs"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RECEPTOR snr 3 density low\n",
" done\n",
"MICROTUBULE snr 1 density low\n",
" done\n",
"RECEPTOR snr 7 density mid\n",
" done\n",
"VESICLE snr 4 density low\n",
" done\n",
"MICROTUBULE snr 4 density mid\n",
" done\n",
"VESICLE snr 1 density mid\n",
" done\n",
"RECEPTOR snr 2 density low\n",
" done\n",
"MICROTUBULE snr 7 density mid\n",
" done\n",
"RECEPTOR snr 1 density low\n",
" done\n",
"VESICLE snr 2 density mid\n",
" done\n",
"MICROTUBULE snr 2 density low\n",
" done\n",
"VESICLE snr 7 density low\n",
" done\n",
"RECEPTOR snr 4 density mid\n",
" done\n",
"RECEPTOR snr 2 density mid\n",
" done\n",
"VESICLE snr 1 density low\n",
" done\n",
"MICROTUBULE snr 4 density low\n",
" done\n",
"RECEPTOR snr 7 density high\n",
" done\n",
"VESICLE snr 2 density high\n",
" done\n",
"MICROTUBULE snr 4 density high\n",
" done\n",
"MICROTUBULE snr 7 density high\n",
" done\n",
"VESICLE snr 4 density mid\n",
" done\n",
"RECEPTOR snr 7 density low\n",
" done\n",
"VESICLE snr 1 density high\n",
" done\n",
"RECEPTOR snr 3 density mid\n",
" done\n",
"MICROTUBULE snr 1 density mid\n",
" done\n",
"RECEPTOR snr 4 density high\n",
" done\n",
"RECEPTOR snr 3 density high\n",
" done\n",
"RECEPTOR snr 4 density low\n",
" done\n",
"RECEPTOR snr 2 density high\n",
" done\n",
"VESICLE snr 7 density mid\n",
" done\n",
"VESICLE snr 7 density high\n",
" done\n",
"MICROTUBULE snr 2 density mid\n",
" done\n",
"MICROTUBULE snr 1 density high\n",
" done\n",
"MICROTUBULE snr 2 density high\n",
" done\n",
"VESICLE snr 4 density high\n",
" done\n",
"RECEPTOR snr 1 density high\n",
" done\n",
"VESICLE snr 2 density low\n",
" done\n",
"RECEPTOR snr 1 density mid\n",
" done\n",
"MICROTUBULE snr 7 density low\n",
" done\n",
"Done all.\n"
]
}
],
"source": [
"R = 3\n",
"frames_for_test_data = np.arange(0,99,10) # 10 out of 100\n",
"\n",
"for xml_file, tif_file in zip(xmls_chosen, tifs_chosen):\n",
" # each tif file is a stack; 100 frames\n",
"\n",
" base_name = os.path.splitext(xml_file)[0]\n",
" print(base_name)\n",
"\n",
" # read\n",
" df = read_xml_as_pands( os.path.join(DATA_DIR, xml_file) )\n",
" img = tifffile.imread( os.path.join(DATA_DIR, tif_file))\n",
"\n",
" img_lbl = df_to_lbl( img, df, radius=R )\n",
" \n",
" # for each frame:\n",
" for _fr, (_im, _lbl) in enumerate(zip(img, img_lbl)):\n",
" _fname = '%s_%03d.tif'%(base_name, _fr)\n",
" \n",
" if _fr in frames_for_test_data:\n",
" # save as TEST DATA\n",
" img_abs_path = os.path.join( TE_IMAGES_DIR, _fname )\n",
" lbl_abs_path = os.path.join( TE_MASKS_DIR, _fname )\n",
" else:\n",
" # save as TRAINING DATA\n",
" img_abs_path = os.path.join( TR_IMAGES_DIR, _fname )\n",
" lbl_abs_path = os.path.join( TR_MASKS_DIR, _fname )\n",
"\n",
" # print( ' ', img_abs_path )\n",
" # print( ' ', lbl_abs_path )\n",
" tifffile.imwrite(img_abs_path, data=_im )\n",
" tifffile.imwrite(lbl_abs_path, data=_lbl )\n",
" \n",
" print(' done')\n",
" \n",
"print('Done all.')"
]
},
{
"cell_type": "code",
"execution_count": 203,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
%% Cell type:markdown id: tags:
### This notebook is to generate training data from the original tifs and xmls
%% Cell type:code id: tags:
``` python
import os, sys
import xml.etree.ElementTree as ET
import pandas as pd
import tifffile
import matplotlib.pyplot as plt
import numpy as np
from skimage.morphology import dilation, selem
ROOT_DIR = os.path.abspath( '..' )
DATA_DIR = os.path.join( ROOT_DIR, 'data' )
DATA_RAW_DIR = os.path.join( DATA_DIR, '1_raw_movies_and_xmls' )
DATA_GEN_DIR = os.path.join(DATA_DIR, '2_generated' )
IMAGES_DIR = os.path.join(DATA_GEN_DIR, 'images' )
MASKS_DIR = os.path.join(DATA_GEN_DIR, 'masks' )
# TR_IMAGES_DIR = os.path.join(TRAINING_DIR, 'images' )
# TR_MASKS_DIR = os.path.join(TRAINING_DIR, 'masks' )
# TEST_DIR = os.path.join(DATA_DIR, 'test' )
# TE_IMAGES_DIR = os.path.join(TEST_DIR, 'images' )
# TE_MASKS_DIR = os.path.join(TEST_DIR, 'masks' )
# for f in [TRAINING_DIR, TEST_DIR, TR_IMAGES_DIR, TR_MASKS_DIR, TE_IMAGES_DIR, TE_MASKS_DIR]:
for f in [IMAGES_DIR, MASKS_DIR]:
if not os.path.exists(f):
os.makedirs(f)
print('Made dir: %s'%f)
```
%%%% Output: stream
Made dir: /master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/images
Made dir: /master/home/dershov/Development/DL/stardist-trackmate/data/2_generated/masks
%% Cell type:code id: tags:
``` python
def read_xml_as_pands( xml_abs_path ):
tree = ET.parse( xml_abs_path )
root = tree.getroot()
particles = root[0]
# the fastes way is to first collect all the data and then feed it to pandas.
# if you create pandas in each loop step, it will be way longer
ii, tt, xx, yy, zz = [], [],[],[],[]
for pi, detections in enumerate(particles): #over particles
for d in detections: # over each p's detectoins
t,x,y,z = d.attrib['t'], d.attrib['x'], d.attrib['y'], d.attrib['z']
ii.append(pi)
tt.append(t)
xx.append(x)
yy.append(y)
zz.append(z)
# feed the collected data to pandas
df = pd.DataFrame(data={'pid':ii, 'fr':tt, 'x':xx, 'y':yy, 'z':zz} )
for n in ['x','y','z']:
df[n] = pd.to_numeric(df[n], downcast='float')
# it is all strings now; convert to numeric
df['fr'] = pd.to_numeric(df['fr'], downcast='integer')
df['pid'] = pd.to_numeric(df['pid'], downcast='integer')
return df
def df_to_lbl( img, df, radius ):
# df = ['pid', 'fr', 'x', 'y', 'z']
id_max = df['pid'].max()
assert id_max < 2**16-1, "Label image is coded to be unsigned 16bit; you have %d particles."%id_max
# dilate with: