#!/bin/bash
#SBATCH -A INA23_C9T05
#SBATCH -p m100_usr_prod
### number of nodes
#SBATCH -N 1
### number of hyperthreading threads
#SBATCH --ntasks-per-core=1
### number of MPI tasks per node
#SBATCH --ntasks-per-node=4
### number of openmp threads
#SBATCH --cpus-per-task=2
### number of allocated GPUs per node
#SBATCH --gres=gpu:4
##SBATCH --mem=200G
#SBATCH -o nodes.out                                                                                                                            #SBATCH -e nodes.err
#SBATCH -t 1:00:00

module purge
module load gnu/10.3.0
module load hpc-sdk/2022--binary
module load spectrum_mpi/10.3.1--binary
module load fftw/3.3.8--spectrum_mpi--10.3.1--binary

#export LD_LIBRARY_PATH=/u/dssc/glacop00/Library_fftw/lib:$LD_LIBRARY_PATH

OUT_SHM=result_intermediate_m100
OUT_SHM_RES=/m100/home/userexternal/glacopo0/LOFAR/hpc_imaging-end-of-re_structuring/scripts/Test_0/m100_1node

rm -f ${OUT_SHM} ${OUT_SHM_RES}

export use_cufftmp=no

if [ "$use_cufftmp" = "yes" ]
then
  export typestring=omp_cufftMp
  export exe=w-stackingC_cufftMp
fi

if [ "$use_cufftmp" = "no" ]
then
  export typestring=omp_gpu_fftw
  export exe=w-stackingCfftw_ring_omp
fi


cd /m100/home/userexternal/glacopo0/LOFAR/hpc_imaging-end-of-re_structuring/
make SYSTYPE=V100_Marconi -j1 clean
rm -f ${exe}
if [ "$use_cufftmp" = "yes" ]
then
    make SYSTYPE=V100_Marconi -j1 mpi_omp_fft
fi

if [ "$use_cufftmp" = "no" ]
then
    make SYSTYPE=V100_Marconi -j1 mpi_ring_omp
fi
    
Hmin=1
Hmax=1
Tmin=4
Tmax=4

for (( h=$Hmin; h<=$Hmax; h*=2 ))
do
    for (( t=$Tmin; t<=$Tmax; t*=2 ))
    do
	N=$(( ${h}*${t} ))
	echo -e "\tRunning $h hosts\n"
	export logdir=mpi_${N}_${typestring}_${SLURM_CPUS_PER_TASK}
	echo "Creating $logdir"
	rm -fr $logdir
	mkdir $logdir

	for itest in {1..3}
	do
	    export logfile=test_${itest}_${logdir}.log
	    echo "time mpirun -np ${N} --map-by ppr:${t}:node /m100/home/userexternal/glacopo0/LOFAR/hpc_imaging-end-of-re_structuring/${exe} /m100/home/userexternal/glacopo0/LOFAR/hpc_imaging-end-of-re_structuring/data/paramfile.txt" > $logfile
	    time /cineca/prod/opt/compilers/hpc-sdk/2022/binary//Linux_ppc64le/2022/comm_libs/mpi/bin/mpirun -np ${N} --map-by ppr:${t}:node --bind-to core -mca btl self,vader /m100/home/userexternal/glacopo0/LOFAR/hpc_imaging-end-of-re_structuring/${exe} data/paramfile.txt >> $logfile
	    mv $logfile $logdir

            
	    Reduce_mpi_time=$( grep -w 'Reduce MPI time:' $logdir/$logfile | gawk '{print $4}' )
            Reduce_shmem_time=$( grep -w 'Reduce Shmem time:' $logdir/$logfile | gawk '{print $4}' )
            FFTW_time=$( grep -w 'FFTW time:' $logdir/$logfile | gawk '{print $3}' )
            Phase_time=$( grep -w 'Phase time:' $logdir/$logfile | gawk '{print $3}' )
            Total_time=$( grep -w 'TOT time:' $logdir/$logfile | gawk '{print $3}' )
            Setup_time=$( grep -w 'Setup time:' $logdir/$logfile | gawk '{print $3}' )
            Kernel_time=$( grep -w 'Kernel time =' $logdir/$logfile | gawk '{print $4}' )
            Array_time=$( grep -w 'Array Composition time' $logdir/$logfile | gawk '{print $8}' )

            echo $N $Reduce_mpi_time $Reduce_shmem_time $FFTW_time $Phase_time $Total_time $Setup_time $Kernel_time $Array_time>> ${OUT_SHM}

            mv timings.dat ${logdir}/timings_${itest}.dat
            cat ${logdir}/timings_all.dat ${logdir}/timings_${itest}.dat >> ${logdir}/timings_all.dat
        done
	
	echo -e "\n\n" >> ${OUT_SHM}
        avg_redmpi=$( awk '{sum+=$2} END { print sum/3 }' ${OUT_SHM} )
        avg_redshmem=$( awk '{sum+=$3} END { print sum/3 }' ${OUT_SHM} )
        avg_fftw=$( awk '{sum+=$4} END { print sum/3 }' ${OUT_SHM} )
        avg_phase=$( awk '{sum+=$5} END { print sum/3 }' ${OUT_SHM} )
        avg_tot=$( awk '{sum+=$6} END { print sum/3 }' ${OUT_SHM} )
        avg_set=$( awk '{sum+=$7} END { print sum/3 }' ${OUT_SHM} )
        avg_ker=$( awk '{sum+=$8} END { print sum/3 }' ${OUT_SHM} )
        avg_arr=$( awk '{sum+=$9} END { print sum/3 }' ${OUT_SHM} )
        std_redmpi=$( awk '{if($2!=""){count++;sum+=$2};y+=$2^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_redshmem=$( awk '{if($3!=""){count++;sum+=$3};y+=$3^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_fftw=$( awk '{if($4!=""){count++;sum+=$4};y+=$4^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_phase=$( awk '{if($5!=""){count++;sum+=$5};y+=$5^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_tot=$( awk '{if($6!=""){count++;sum+=$6};y+=$6^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_set=$( awk '{if($7!=""){count++;sum+=$7};y+=$7^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_ker=$( awk '{if($8!=""){count++;sum+=$8};y+=$8^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )
        std_arr=$( awk '{if($9!=""){count++;sum+=$9};y+=$9^2} END{sq=sqrt(y/3-(sum/3)^2);sq=sq?sq:0;print sq}' ${OUT_SHM} )

        echo $N $avg_redmpi $std_redmpi $avg_redshmem $std_redshmem $avg_fftw $std_fftw $avg_phase $std_phase $avg_tot $std_tot $avg_set $std_set $avg_ker $std_ker $avg_arr $std_arr  >> ${OUT_SHM_RES}
        rm -f ${OUT_SHM}
    done
    echo -e "\n\n" >> ${OUT_SHM_RES}
done
