NUMA awarness

Makefile

@@ -30,7 +30,7 @@ endif
 
 #OPT += -DNVIDIA
 # perform one-side communication (suggested) instead of reduce (only if MPI is active)
-#OPT += -DONE_SIDE
+OPT += -DONE_SIDE
 # write the full 3D cube of gridded visibilities and its FFT transform
 #OPT += -DWRITE_DATA
 # write the final image
@@ -39,8 +39,8 @@ OPT += -DWRITE_IMAGE
 OPT += -DPHASE_ON
 
 
-DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c
-COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o
+DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c numa.h
+COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o numa.o
 
 w-stacking.c: w-stacking.cu
 	cp w-stacking.cu w-stacking.c

allvars.c

@@ -27,6 +27,7 @@ long * histo_send, size_of_grid;
 double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w;
 
 #ifdef USE_MPI
+      MPI_Comm MYMPI_COMM_WORLD;
       MPI_Win slabwin;
 #endif
 

allvars.h

@@ -27,6 +27,8 @@
 #include <math.h>
 #include <time.h>
 #include <unistd.h>
+#include "numa.h"
+
 
 extern struct io
 {
@@ -128,6 +130,7 @@ extern long * histo_send, size_of_grid;
 extern double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w;
 
 #ifdef USE_MPI
+    extern MPI_Comm MYMPI_COMM_WORLD;
     extern  MPI_Win slabwin;
 #endif
 

gridding.c

@@ -248,6 +248,7 @@ void gridding_data(){
        
       #ifdef ONE_SIDE
        printf("One Side communication active\n");
+       memcpy(Me.win.ptr, gridss, size_of_grid);
        MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
        MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
        MPI_Win_unlock(target_rank,slabwin);

init.c

@@ -30,6 +30,10 @@ void init(int index)
    xaxis = local_grid_size_x;
    yaxis = local_grid_size_y;
 
+   #ifdef USE_MPI
+   	init_numa( rank, size, &MYMPI_COMM_WORLD, &Me );
+   #endif
+
    clock_gettime(CLOCK_MONOTONIC, &begin);
    start = clock();
 

main.c

@@ -28,6 +28,7 @@ int main(int argc, char * argv[])
 		#ifdef USE_FFTW
 			fftw_mpi_init();
   		#endif
+                MPI_Comm_dup(MPI_COMM_WORLD, &MYMPI_COMM_WORLD);
 	#else
 		rank = 0;
 		size = 1;

proto.h

 /* function declaration */
 
 
+/* numa.c */
+int init_numa( int, int, MPI_Comm *, map_t *);
+int shutdown_numa( int, int, MPI_Comm *, map_t *);
+
 /* init.c */
 
 void init(int i);