From b1daf9884290140180ad9aaefc83478bcd0bdefe Mon Sep 17 00:00:00 2001
From: nandhanas <s.nandhana@gmail.com>
Date: Mon, 21 Mar 2022 12:20:49 +0100
Subject: [PATCH] NUMA awarness

---
 gridding.c | 47 +++++++++++++++++++++++++++++++++++++++++------
 init.c     |  3 +++
 proto.h    |  2 ++
 3 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/gridding.c b/gridding.c
index de4cc6e..1e34d60 100644
--- a/gridding.c
+++ b/gridding.c
@@ -91,7 +91,7 @@ void initialize_array(){
 void gridding_data(){
 
     double shift = (double)(dx*yaxis);
-    
+ 
     // Open the MPI Memory Window for the slab
     #ifdef USE_MPI
         MPI_Win_create(grid, size_of_grid*sizeof(double), sizeof(double), MPI_INFO_NULL, MPI_COMM_WORLD, &slabwin);
@@ -248,11 +248,16 @@ void gridding_data(){
        
       #ifdef ONE_SIDE
        printf("One Side communication active\n");
-       memcpy(Me.win.ptr, gridss, size_of_grid);
-       MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
-       MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
-       MPI_Win_unlock(target_rank,slabwin);
-               //MPI_Put(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,slabwin);
+    //   MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
+      // MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
+      // MPI_Win_unlock(target_rank,slabwin);
+       memcpy(Me.win.ptr+isector*sizeof(gridss), gridss, sizeof(gridss));
+       MPI_Barrier(MPI_COMM_WORLD);
+       printf("rank %d\n", rank);
+       reduce(target_rank, isector, 0, size-1);
+       if((Me.MAXl > myHOST) && (Me.Rank[HOSTS] != -1))
+         MPI_Reduce(grid,grid,size_of_grid,MPI_DOUBLE,MPI_SUM,target_rank,*Me.COMM[HOSTS]);  
+       //MPI_Put(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,slabwin);
       #else
        MPI_Reduce(gridss,grid,size_of_grid,MPI_DOUBLE,MPI_SUM,target_rank,MPI_COMM_WORLD);
       #endif //ONE_SIDE
@@ -378,3 +383,33 @@ void write_grided_data()
    #endif //WRITE_DATA 
 
 }
+
+void reduce(int target, int sector, int begin, int end)
+{
+   if(begin >= end)
+       return;
+   int mid = (begin+end)/2;
+   reduce(target, sector, begin, mid);
+   reduce(target, sector, mid+1, end);
+   accumulate(target, sector, begin, mid, end);
+}
+
+
+void accumulate(int target, int sector, int begin, int mid, int end)
+{
+
+    for(int i =mid; i<end; i++)
+    {
+        if(i != target)
+        {
+             for(int j=0; j<size_of_grid; j++)
+             {
+                    *((double*)Me.swins[target].ptr+sector*sizeof(gridss)+j) += *((double*)Me.swins[i].ptr+sector*sizeof(gridss)+j);
+                    //printf("value = %lf, rank = %d\n", *((double*)Me.swins[target].ptr+isector*sizeof(gridss)+j), rank);
+             }
+        }
+     }
+    //if(rank == target_rank)
+    //   memcpy(grid, Me.swins[target_rank].ptr+isector*sizeof(gridss), sizeof(grid));    
+
+}
diff --git a/init.c b/init.c
index fc9297d..1f8b081 100644
--- a/init.c
+++ b/init.c
@@ -32,6 +32,9 @@ void init(int index)
 
    #ifdef USE_MPI
    	init_numa( rank, size, &MYMPI_COMM_WORLD, &Me );
+
+        if(rank == 0)
+    	    printf("\nTask %d sees %d topology levels\n", rank, Me.MAXl);
    #endif
 
    clock_gettime(CLOCK_MONOTONIC, &begin);
diff --git a/proto.h b/proto.h
index 5b6aadc..353b6a7 100644
--- a/proto.h
+++ b/proto.h
@@ -23,6 +23,8 @@ void gridding();
 void initialize_array();
 void gridding_data();
 void write_grided_data();
+void reduce(int target, int sector, int begin, int end);
+void accumulate(int target, int sector, int begin, int mid, int end);
 
 
 /* fourier_transform.c */
-- 
GitLab