NUMA awarness

gridding.c

@@ -248,10 +248,15 @@ void gridding_data(){
        
       #ifdef ONE_SIDE
        printf("One Side communication active\n");
-       memcpy(Me.win.ptr, gridss, size_of_grid);
-       MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
-       MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
-       MPI_Win_unlock(target_rank,slabwin);
+    //   MPI_Win_lock(MPI_LOCK_SHARED,target_rank,0,slabwin);
+      // MPI_Accumulate(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,MPI_SUM,slabwin);
+      // MPI_Win_unlock(target_rank,slabwin);
+       memcpy(Me.win.ptr+isector*sizeof(gridss), gridss, sizeof(gridss));
+       MPI_Barrier(MPI_COMM_WORLD);
+       printf("rank %d\n", rank);
+       reduce(target_rank, isector, 0, size-1);
+       if((Me.MAXl > myHOST) && (Me.Rank[HOSTS] != -1))
+         MPI_Reduce(grid,grid,size_of_grid,MPI_DOUBLE,MPI_SUM,target_rank,*Me.COMM[HOSTS]);  
        //MPI_Put(gridss,size_of_grid,MPI_DOUBLE,target_rank,0,size_of_grid,MPI_DOUBLE,slabwin);
       #else
        MPI_Reduce(gridss,grid,size_of_grid,MPI_DOUBLE,MPI_SUM,target_rank,MPI_COMM_WORLD);
@@ -378,3 +383,33 @@ void write_grided_data()
    #endif //WRITE_DATA 
 
 }
+
+void reduce(int target, int sector, int begin, int end)
+{
+   if(begin >= end)
+       return;
+   int mid = (begin+end)/2;
+   reduce(target, sector, begin, mid);
+   reduce(target, sector, mid+1, end);
+   accumulate(target, sector, begin, mid, end);
+}
+
+
+void accumulate(int target, int sector, int begin, int mid, int end)
+{
+
+    for(int i =mid; i<end; i++)
+    {
+        if(i != target)
+        {
+             for(int j=0; j<size_of_grid; j++)
+             {
+                    *((double*)Me.swins[target].ptr+sector*sizeof(gridss)+j) += *((double*)Me.swins[i].ptr+sector*sizeof(gridss)+j);
+                    //printf("value = %lf, rank = %d\n", *((double*)Me.swins[target].ptr+isector*sizeof(gridss)+j), rank);
+             }
+        }
+     }
+    //if(rank == target_rank)
+    //   memcpy(grid, Me.swins[target_rank].ptr+isector*sizeof(gridss), sizeof(grid));    
+
+}

init.c

@@ -32,6 +32,9 @@ void init(int index)
 
    #ifdef USE_MPI
    	init_numa( rank, size, &MYMPI_COMM_WORLD, &Me );
+
+        if(rank == 0)
+    	    printf("\nTask %d sees %d topology levels\n", rank, Me.MAXl);
    #endif
 
    clock_gettime(CLOCK_MONOTONIC, &begin);

proto.h

@@ -23,6 +23,8 @@ void gridding();
 void initialize_array();
 void gridding_data();
 void write_grided_data();
+void reduce(int target, int sector, int begin, int end);
+void accumulate(int target, int sector, int begin, int mid, int end);
 
 
 /* fourier_transform.c */