Edit leonardo_booster.sh

leonardo_booster.sh

@@ -4,46 +4,45 @@
 #SBATCH --cpus-per-task 32
 #SBATCH --gres=gpu:1
 #SBATCH --partition=boost_usr_prod
-
-set -xev #exit on error
+set -xev
 
 export DEBUG=1
+export NV_ACC_CUDA_STACKSIZE=15000
 
-PPL=(1 64) #vary particle per leaf
-THREADS=(1 2 4 8 16 32 64 128) #vary branching (thread) count
-MODE=(cpu gpu) #vary gpu/cpu switch
-#N=(1000 10000 100000 1000000 10000000)
-N=(50000000) #scaling will use 5e7 particles
+PPL=(1 8 16 32)
+THREADS=(1 2 4 8 16 32)
+MODE=(gpu cpu)
+N=(1e7)
+OMP_GPU=100 #number of particles per GPU thread
 
 #
-# run tree build tests on GPU
+# test tree build on GPU and CPU
 #
-CC='nvc++' HW_FLAGS='-mp=gpu -gpu=rdc,managed -gpu=cc80  -Minfo=mp,accel -Minline=1000 -O3 -DTARGET=1 -DOPENMP_RECURSION_TRICK -DOPENMP_TARGET_HILBERTCOPY' srun -n 1 python -m hotwheels_core.wrap hotwheels_octree/test_target.cpp
+OMP_NUM_THREADS=4 CC='gcc' HW_FLAGS='-fopenmp -O3' srun -n 1 python -m hotwheels_core.wrap src/test_target.cpp
+OMP_NUM_THREADS=4 CC='nvc++' HW_FLAGS='-mp=gpu -gpu=rdc,managed -gpu=cc80  -Minfo=mp,accel -Minline=10000 -O3' srun -n 1 python -m hotwheels_core.wrap src/test_target.cpp
 
-rm -f report.csv
-#
-# loop over tree build configurations PPL, THREADS, MODE, N
-#
-for ppl in "${PPL[@]}" ; do
-    for n in "${N[@]}" ; do
-        for mode in "${MODE[@]}" ; do
-            for threads in "${THREADS[@]}" ; do
+for ppl in "${PPL[@]}" ; do # loop over particles per leaf
+    for n in "${N[@]}" ; do # loop over n of particles
+        for mode in "${MODE[@]}" ; do #switch between gpu and gpu mode
+            for threads in "${THREADS[@]}" ; do #loop over number of OpenMPThreads
                 if [[ "$mode" == "cpu"  ]]; then
                     export CC=gcc
                     export HW_FLAGS='-O3 -fopenmp'
                     export OMP_NUM_THREADS=$threads
                 else
-                    if [[ $threads -lt 16 ]]; then
+                    if [[ $threads != 8 ]]; then
                        continue
                     fi
                     export CC=nvc++
                     export HW_FLAGS='-mp=gpu -gpu=rdc,managed -gpu=cc80  -Minfo=mp,accel -Minline=1000 -O3'
-                    export OMP_NUM_THREADS=4
+                    export OMP_NUM_THREADS=4 # gpu always uses 4 threads
+                    threads=$(python -c "print(int($N//100))") #for the text hilight: "
                 fi
-                # rnu the test
-                srun -n 1 python hotwheels_octree/testgpu.py -ppl $ppl -n $n -mode $mode -threads $threads 1>run.out 2>run.err
+                # run thre tree build
+                srun -n 1 python src/testgpu.py -ppl $ppl -n $n -mode $mode -threads $threads -defrag 1 1>run.out 2>run.err
+                # grep the timer and store
                 t=$(grep time: run.out | sed s/time://)
-                echo $ppl,$n,$mode,$threads,$t >> report.csv # store the report
+                echo $ppl $n $mode $threads $t >> report.csv
             done
         done
     done