recursive_gmconvert.sh 3.55 KB
Newer Older
samuel hanot's avatar
samuel hanot committed
1
2
#!/bin/zsh

samuel hanot's avatar
samuel hanot committed
3
4
5
6
7
8
9
usage() {
	echo >&2 \
	"usage: recursive_gmconvert.sh -f map_filename -t threshold -n num_gaussians -N num_iterations [-i first_iteration] [-s]
Required arguments:
-f map_filename: the file name of the input EM map
-t threshold: the density threshold
-n num_gaussians: number of gaussians per sub-process
10
-0 num_gaussians0: number of gaussians for first iteration
samuel hanot's avatar
samuel hanot committed
11
12
13
14
15
-N num_iterations: number of recursion levels
Optional arguments:
-i first_iteration: initial recursion level (defaults to 1)
-s: enable serial mode
-h: display this message"
samuel hanot's avatar
samuel hanot committed
16
17
}

samuel hanot's avatar
samuel hanot committed
18
19
20
21
#defaults
map_name=
threshold=
n_gaussians=
22
n_gaussians0=
samuel hanot's avatar
samuel hanot committed
23
N=
samuel hanot's avatar
samuel hanot committed
24
i0=1
samuel hanot's avatar
samuel hanot committed
25
serial=0
samuel hanot's avatar
samuel hanot committed
26

27
while getopts f:t:n:N:i:0:sh opt
samuel hanot's avatar
samuel hanot committed
28
29
30
31
32
do
    case "$opt" in
      f)  map_name="$OPTARG";;
			t)  threshold="$OPTARG";;
			n)  n_gaussians="$OPTARG";;
33
			0)  n_gaussians0="$OPTARG";;
samuel hanot's avatar
samuel hanot committed
34
35
36
37
38
39
40
41
42
43
44
			N)  N="$OPTARG";;
			i)  i0="$OPTARG";;
			s)  serial=1;;
			h)  usage
					exit ;;
      \?)	# unknown flag
      	  echo >&2 \
					usage
				  exit 1;;
    esac
done
samuel hanot's avatar
samuel hanot committed
45

samuel hanot's avatar
samuel hanot committed
46
47
48
49
50
#check that necessary options were set
if [[ ! -e "$map_name" ]] || [[ -z "$threshold" ]] || [[ -z "$n_gaussians" ]] || [[ -z "$N" ]]
then
	usage
	exit 1
samuel hanot's avatar
samuel hanot committed
51
52
fi

samuel hanot's avatar
samuel hanot committed
53
54
55
56
57
if [[ -z "$n_gaussians0" ]]
then
	n_gaussians0=$n_gaussians
fi

samuel hanot's avatar
samuel hanot committed
58
59
60
61
62
63
MaxArraySize=100000
get_max_jobs(){
	echo ${MaxArraySize}
}
profile_cmd="time"

samuel hanot's avatar
samuel hanot committed
64
65
66
67
bindir=${0:h}
impdir=~/imp-fast/
gmconvert=${bindir}/gmconvert/gmconvert

68
69
70
71
72
73
74
init_dir=$PWD
map_root_dir=$(dirname $map_name)
map_name=$(basename $map_name)
cd $map_root_dir



samuel hanot's avatar
samuel hanot committed
75
76
77
mkdir -p converged
for ((i=$i0 ; i<=N ; i++))
do
samuel hanot's avatar
samuel hanot committed
78
	echo "i=$i"
samuel hanot's avatar
samuel hanot committed
79
80
81
82
83
84
	n=$((i))
	jobname=${map_name}_${n}

	n_jobs=1
	if ((i==1))
	then
samuel hanot's avatar
samuel hanot committed
85
86
87
88
		if [[ ! -e threshold.map ]]
		then
			${gmconvert} -imap ${map_name} -zth ${threshold} -oimap threshold.map -ogmm /dev/null -ng 0
	  fi
89
		echo ${profile_cmd} ${gmconvert} -imap ${map_name} -ogmm $n/${n}_0.gmm -ng ${n_gaussians0} -zth ${threshold}
samuel hanot's avatar
samuel hanot committed
90
91
92
93
94
95
96
97
98
99
		echo ${gmconvert}  -igmm $n/${n}_0.gmm -imap ${map_name} -omap /dev/null
	else
		n_prev=$((i-1))
		gmm_name=${n_prev}/${n_prev}.gmm
		n_jobs=0
		for gmm_file in $(cat ${n_prev}/not_converged.txt)
		do
			n_jobs=$((n_jobs + $(awk '/NGAUSS/{s+=$3}END{print s}' ${gmm_file}) ))
		done
		echo echo ${gmm_name} '$((SLURM_ARRAY_TASK_ID-1+offset))' \> 'tmp_$((SLURM_ARRAY_TASK_ID+offset)).txt'  
samuel hanot's avatar
samuel hanot committed
100
101
		echo ${profile_cmd} ${gmconvert} -imap ${map_name}  -gmml 'tmp_$((SLURM_ARRAY_TASK_ID+offset)).txt' -ogmm $n/${n}_'$((SLURM_ARRAY_TASK_ID-1+offset))'.gmm -ng ${n_gaussians} -zth ${threshold}
		echo ${gmconvert}  -igmm $n/${n}_'$((SLURM_ARRAY_TASK_ID-1+offset)).gmm' -imap ${map_name}  -gmml 'tmp_$((SLURM_ARRAY_TASK_ID+offset)).txt' -omap /dev/null
samuel hanot's avatar
samuel hanot committed
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
	fi > commands_$i.sh
	mkdir -p $n
	max_jobs=$(get_max_jobs)
	if ((serial==0))
	then
		for ((j=0 ; j<n_jobs ; j=j+max_jobs))
		do
			jobfile=input_${jobname}_${j}.sh
			k=$max_jobs
			if ((j+k > n_jobs))
			then
				k=$((n_jobs - j))
			fi
			echo j=$j, k=$k, jobfile=$jobfile, n_jobs=${n_jobs}
			${bindir}/make_array_from_cmds.sh $jobname $k $j < commands_$i.sh > $jobfile
			date
			sbatch -o ${n}/"slurm-%a_${j}.out" $jobfile
		done
		jobids=$(squeue -O arrayjobid -u $(whoami) -n ${jobname} -S i -h | tr '\n' ':' | tr -d ' ')
		jobids=${jobids%?}
		echo waiting for jobids=$jobids
		#not ok is because gmconvert exits 1
		srun --job-name 'post' --dependency=afternotok:${jobids} --export=ALL,map_name=${map_name},n=${n},bindir=${bindir} --pty ${bindir}/postprocess.sh
	else
		for ((j=1 ; j<=n_jobs ; j++))
		do
			echo $j/$n_jobs
			SLURM_ARRAY_TASK_ID=$j; offset=0; source commands_$i.sh > ${n}/"slurm-${j}_0.out" 2>&1
		done
		source ${bindir}/postprocess.sh 
	fi
	mkdir -p plots
134
	gnuplot ${bindir}/plot_fsc.gpl > /dev/null 2>&1
samuel hanot's avatar
samuel hanot committed
135
	rm -f tmp*
samuel hanot's avatar
samuel hanot committed
136
done
137
cd $init_dir