diff --git a/Makefile b/Makefile index 6b529a5f965f84d634c6922264e190e6d6c92f8c..318842cba5834abd6b2d18df7a959d61c0eea0ef 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ endif #OPT += -DNVIDIA # perform one-side communication (suggested) instead of reduce (only if MPI is active) -#OPT += -DONE_SIDE +OPT += -DONE_SIDE # write the full 3D cube of gridded visibilities and its FFT transform #OPT += -DWRITE_DATA # write the final image @@ -39,8 +39,8 @@ OPT += -DWRITE_IMAGE OPT += -DPHASE_ON -DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c -COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o +DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c numa.h +COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o numa.o w-stacking.c: w-stacking.cu cp w-stacking.cu w-stacking.c diff --git a/allvars.c b/allvars.c index 95d09f53777038f4830cceb17c184a368e2908e1..8e67a3c12c5d51c0b102f1c61e060b662a447196 100644 --- a/allvars.c +++ b/allvars.c @@ -27,7 +27,8 @@ long * histo_send, size_of_grid; double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w; #ifdef USE_MPI - MPI_Win slabwin; + MPI_Comm MYMPI_COMM_WORLD; + MPI_Win slabwin; #endif long **sectorarray; diff --git a/allvars.h b/allvars.h index 4edb09c690204193e7a01b3cd2fdda241ef595af..2f66fc12b14ddf063a573cbf284f6aab6f4bc02b 100644 --- a/allvars.h +++ b/allvars.h @@ -27,6 +27,8 @@ #include #include #include +#include "numa.h" + extern struct io { @@ -128,6 +130,7 @@ extern long * histo_send, size_of_grid; extern double * grid, *gridss, *gridss_real, *gridss_img, *gridss_w; #ifdef USE_MPI + extern MPI_Comm MYMPI_COMM_WORLD; extern MPI_Win slabwin; #endif diff --git a/gridding.c b/gridding.c index 1b57c94986db0e5ff906e86d03491b4f46c99de6..de4cc6e338f79e41fac112773c85bf416419839d 100644 --- a/gridding.c +++ b/gridding.c @@ -248,6 +248,7 @@ void gridding_data(){ #ifdef ONE_SIDE printf("One Side communication active\n"); + memcpy(Me.win.ptr, gridss, size_of_grid); MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin); MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin); MPI_Win_unlock(target_rank,slabwin); diff --git a/init.c b/init.c index 35711ccffe1a610e1f92e28d462dd7b3a2d27b91..fc9297d64f5c0f3a7c577c54ae592904adeff14d 100644 --- a/init.c +++ b/init.c @@ -30,6 +30,10 @@ void init(int index) xaxis = local_grid_size_x; yaxis = local_grid_size_y; + #ifdef USE_MPI + init_numa( rank, size, &MYMPI_COMM_WORLD, &Me ); + #endif + clock_gettime(CLOCK_MONOTONIC, &begin); start = clock(); diff --git a/main.c b/main.c index c2fb72ae039c02ae61b5f95b0ed78acb2042a3c8..6bf7d013f4e90f1b191aad3ab50881837b83f458 100644 --- a/main.c +++ b/main.c @@ -28,6 +28,7 @@ int main(int argc, char * argv[]) #ifdef USE_FFTW fftw_mpi_init(); #endif + MPI_Comm_dup(MPI_COMM_WORLD, &MYMPI_COMM_WORLD); #else rank = 0; size = 1; diff --git a/proto.h b/proto.h index 977f2f03958392f7512ba72eee1ba285b3a6f602..5b6aadc62545976abe7ccb2662158e1c91d0c1be 100644 --- a/proto.h +++ b/proto.h @@ -1,6 +1,10 @@ /* function declaration */ +/* numa.c */ +int init_numa( int, int, MPI_Comm *, map_t *); +int shutdown_numa( int, int, MPI_Comm *, map_t *); + /* init.c */ void init(int i);