From 624fdbdbf7124ad8367e43b3b7510e45332d96b9 Mon Sep 17 00:00:00 2001
From: nandhanas <s.nandhana@gmail.com>
Date: Tue, 8 Mar 2022 18:16:26 +0100
Subject: [PATCH] NUMA awarness

---
 Makefile   | 6 +++---
 allvars.c  | 3 ++-
 allvars.h  | 3 +++
 gridding.c | 1 +
 init.c     | 4 ++++
 main.c     | 1 +
 proto.h    | 4 ++++
 7 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6b529a5..318842c 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@ endif
 
 #OPT += -DNVIDIA
 # perform one-side communication (suggested) instead of reduce (only if MPI is active)
-#OPT += -DONE_SIDE
+OPT += -DONE_SIDE
 # write the full 3D cube of gridded visibilities and its FFT transform
 #OPT += -DWRITE_DATA
 # write the final image
@@ -39,8 +39,8 @@ OPT += -DWRITE_IMAGE
 OPT += -DPHASE_ON
 
 
-DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c
-COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o
+DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c numa.h
+COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o numa.o
 
 w-stacking.c: w-stacking.cu
 	cp w-stacking.cu w-stacking.c
diff --git a/allvars.c b/allvars.c
index 95d09f5..8e67a3c 100644
--- a/allvars.c
+++ b/allvars.c
@@ -27,7 +27,8 @@ long * histo_send, size_of_grid;
 double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w;
 
 #ifdef USE_MPI
-    MPI_Win slabwin;
+      MPI_Comm MYMPI_COMM_WORLD;
+      MPI_Win slabwin;
 #endif
 
 long **sectorarray;
diff --git a/allvars.h b/allvars.h
index 4edb09c..2f66fc1 100644
--- a/allvars.h
+++ b/allvars.h
@@ -27,6 +27,8 @@
 #include <math.h>
 #include <time.h>
 #include <unistd.h>
+#include "numa.h"
+
 
 extern struct io
 {
@@ -128,6 +130,7 @@ extern long * histo_send, size_of_grid;
 extern double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w;
 
 #ifdef USE_MPI
+    extern MPI_Comm MYMPI_COMM_WORLD;
     extern  MPI_Win slabwin;
 #endif
 
diff --git a/gridding.c b/gridding.c
index 1b57c94..de4cc6e 100644
--- a/gridding.c
+++ b/gridding.c
@@ -248,6 +248,7 @@ void gridding_data(){
        
       #ifdef ONE_SIDE
        printf("One Side communication active\n");
+       memcpy(Me.win.ptr, gridss, size_of_grid);
        MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
        MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
        MPI_Win_unlock(target_rank,slabwin);
diff --git a/init.c b/init.c
index 35711cc..fc9297d 100644
--- a/init.c
+++ b/init.c
@@ -30,6 +30,10 @@ void init(int index)
    xaxis = local_grid_size_x;
    yaxis = local_grid_size_y;
 
+   #ifdef USE_MPI
+   	init_numa( rank, size, &MYMPI_COMM_WORLD, &Me );
+   #endif
+
    clock_gettime(CLOCK_MONOTONIC, &begin);
    start = clock();
 
diff --git a/main.c b/main.c
index c2fb72a..6bf7d01 100644
--- a/main.c
+++ b/main.c
@@ -28,6 +28,7 @@ int main(int argc, char * argv[])
 		#ifdef USE_FFTW
 			fftw_mpi_init();
   		#endif
+                MPI_Comm_dup(MPI_COMM_WORLD, &MYMPI_COMM_WORLD);
 	#else
 		rank = 0;
 		size = 1;
diff --git a/proto.h b/proto.h
index 977f2f0..5b6aadc 100644
--- a/proto.h
+++ b/proto.h
@@ -1,6 +1,10 @@
 /* function declaration */
 
 
+/* numa.c */
+int init_numa( int, int, MPI_Comm *, map_t *);
+int shutdown_numa( int, int, MPI_Comm *, map_t *);
+
 /* init.c */
 
 void init(int i);
-- 
GitLab