#!/bin/bash

#SBATCH --job-name=LOFAR
#SBATCH --partition=t-x2-lw
#SBATCH --account=space
#SBATCH --nodes=4
#SBATCH --nodelist=tcnode01
#SBATCH --tasks=256
##SBATCH --mem=100000
##SBATCH -w len42
#SBATCH --ntasks-per-node=64
#SBATCH --cpus-per-task=1
#SBATCH -o nodes.out
#SBATCH -e nodes.err
#SBATCH --time=01:00:00

module load gcc-8.5.0/ompi-4.1.4_wo-cuda

export LD_LIBRARY_PATH=/home/giacopo/Library_fftw/lib:$LD_LIBRARY_PATH

cd /home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/
make SYSTYPE=Thunder -j1 clean
rm -f w-stackingCfftw_ring
make SYSTYPE=Thunder -j1 mpi_ring

OUT_SHM=result_thunder
OUT_SHM_RES=/home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/scripts/Test_0/results_thunder_nodes

rm -f ${OUT_SHM} ${OUT_SHM_RES}

export use_cuda=no

if [ "$use_cuda" = "no" ]
then
  export typestring=omp_cpu
  export exe=w-stackingCfftw_ring
fi

if [ "$use_cuda" = "yes" ]
then
  export typestring=cuda
  export exe=w-stackingfftw
fi

Hmin=2
Hmax=4
Tmin=64
Tmax=64

for (( h=$Hmin; h<=$Hmax; h*=2 ))
do
    for (( t=$Tmin; t<=$Tmax; t*=2 ))
    do
	N=$(( ${h}*${t} ))
	echo -e "\tRunning $h hosts\n"
	export logdir=mpi_${N}_${typestring}_${SLURM_CPUS_PER_TASK}
	echo "Creating $logdir"
	rm -fr $logdir
	mkdir $logdir

	for itest in {1..3}
	do
	    export logfile=test_${itest}_${logdir}.log
	    echo "time mpirun -np ${N} --map-by ppr:${t}:node /home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/${exe} /home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/data/paramfile.txt" > $logfile
	    time mpirun -np ${N} --map-by ppr:${t}:node --bind-to core -mca pml ucx -x UCX_NET_DEVICES=mlx5_1:1 /home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/${exe} /home/giacopo/LOFAR/hpc_imaging-end-of-re_structuring/data/paramfile.txt >> $logfile
	    mv $logfile $logdir

           
	    Reduce_time=$( grep -w 'Reduce time:' $logdir/$logfile | gawk '{print $11}' )
            FFTW_time=$( grep -w 'FFTW time:' $logdir/$logfile | gawk '{print $3}' )
            Phase_time=$( grep -w 'Phase time:' $logdir/$logfile | gawk '{print $3}' )
            Total_time=$( grep -w 'TOT time:' $logdir/$logfile | gawk '{print $3}' )

	    echo $N $Reduce_time $FFTW_time $Phase_time $Total_time >> ${OUT_SHM}
		
	    mv timings.dat ${logdir}/timings_${itest}.dat
	    cat ${logdir}/timings_all.dat ${logdir}/timings_${itest}.dat >> ${logdir}/timings_all.dat
	done

	echo -e "\n\n" >> ${OUT_SHM}
	avg_red=$( awk '{sum+=$2} END { print sum/3 }' ${OUT_SHM} )
        avg_fftw=$( awk '{sum+=$3} END { print sum/3 }' ${OUT_SHM} )
        avg_phase=$( awk '{sum+=$4} END { print sum/3 }' ${OUT_SHM} )
        avg_tot=$( awk '{sum+=$5} END { print sum/3 }' ${OUT_SHM} )
        std_red=$( awk '{if($2!=""){count++;sum+=$2};y+=$2^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_fftw=$( awk '{if($3!=""){count++;sum+=$3};y+=$3^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_phase=$( awk '{if($4!=""){count++;sum+=$4};y+=$4^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_tot=$( awk '{if($5!=""){count++;sum+=$5};y+=$5^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )

        echo $N $avg_red $std_red $avg_fftw $std_fftw $avg_phase $std_phase $avg_tot $std_tot >> ${OUT_SHM_RES}
        rm -f ${OUT_SHM}
    done
    echo -e "\n\n" >> ${OUT_SHM_RES}
done
