#!/bin/bash
##SBATCH -A clusters
#SBATCH -p EPYC
#SBATCH -J LOFAR
### number of nodes
#SBATCH -N 2
### number of hyperthreading threads
#SBATCH --ntasks-per-core=1
### number of MPI tasks per node
#SBATCH --ntasks-per-node=128
#SBATCH -n 256
### number of openmp threads
#SBATCH --cpus-per-task=1
### number of allocated GPUs per node
##SBATCH --gres=gpu:2
#SBATCH --mem=490G
#SBATCH -o test.out
#SBATCH -e test.err
#SBATCH -t 02:00:00 

module purge
module load architecture/AMD
module load compiler
module load openMPI/4.1.4/gnu/12.2.1

export LD_LIBRARY_PATH=/u/dssc/glacop00/Library_fftw/lib:$LD_LIBRARY_PATH

cd /u/dssc/glacop00/LOFAR/hpc_imaging-end-of-re_structuring/
make SYSTYPE=Epyc -j1 clean
rm -f w-stackingCfftw_ring
make SYSTYPE=Epyc -j1 mpi_ring

export use_cuda=no

if [ "$use_cuda" = "no" ]
then
  export typestring=omp_cpu
  export exe=w-stackingCfftw_ring
fi

if [ "$use_cuda" = "yes" ]
then
  export typestring=cuda
  export exe=w-stackingfftw
fi

Hmin=1
Hmax=2
Tmin=128
Tmax=128

for (( h=$Hmin; h<=$Hmax; h*=2 ))
do
    for (( t=$Tmin; t<=$Tmax; t*=2 ))
    do
	N=$(( ${h}*${t} ))
	echo -e "\tRunning $h hosts\n"
	export logdir=mpi_${N}_${typestring}_${SLURM_CPUS_PER_TASK}
	echo "Creating $logdir"
	rm -fr $logdir
	mkdir $logdir

	for itest in {1..2}
	do
	    export logfile=test_${itest}_${logdir}.log
	    echo "time mpirun -np ${N} --map-by ppr:${t}:node /u/dssc/glacop00/LOFAR/hpc_imaging-end-of-re_structuring/${exe} /u/dssc/glacop00/LOFAR/hpc_imaging-end-of-re_structuring/data/paramfile.txt" > $logfile
	    time mpirun -np ${N} --map-by ppr:${t}:node --bind-to core --mca btl self,vader /u/dssc/glacop00/LOFAR/hpc_imaging-end-of-re_structuring/${exe} /u/dssc/glacop00/LOFAR/hpc_imaging-end-of-re_structuring/data/paramfile.txt >> $logfile
	    mv $logfile $logdir
	    mv timings.dat ${logdir}/timings_${itest}.dat
	    cat ${logdir}/timings_all.dat ${logdir}/timings_${itest}.dat >> ${logdir}/timings_all.dat
	done
    done
done
