diff --git a/allvars.c b/allvars.c index d391fe02e561c930dc75cc1b17d1054ac1c5ed44..92c25190baa76b2e998d7f92816f286fea44d489 100644 --- a/allvars.c +++ b/allvars.c @@ -11,5 +11,14 @@ struct time timing; char filename[1000]; int num_threads; -char datapath_multi[NFILES][900]; +char datapath_multi[NFILES][900], datapath[900]; +int xaxis, yaxis; int ndatasets; +int grid_size_x = 2048; +int grid_size_y = 2048; +int num_w_planes = 8; +int rank; +int size; +clock_t start, end, start0, startk, endk; +struct timespec begin, finish, begin0, begink, finishk; +long nsectors; diff --git a/allvars.h b/allvars.h index 03220e51dfc4eaa9f2dba7f75c23f1f007afdb87..98375262eb7af9a7b9848681e5d3606fc12e79aa 100644 --- a/allvars.h +++ b/allvars.h @@ -1,7 +1,32 @@ /* file to store global variables*/ #include +#include +#include +#ifdef USE_MPI +#include +#ifdef USE_FFTW +#include +#endif +#endif +#ifdef ACCOMP +#include "w-stacking_omp.h" +#else +#include "w-stacking.h" +#endif +#ifdef NVIDIA +#include +#endif +#define PI 3.14159265359 +#define NUM_OF_SECTORS -1 +#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) +#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) +#define NOVERBOSE #define NFILES 100 +#include +#include +#include +#include extern struct io { @@ -68,5 +93,15 @@ extern struct time extern char filename[1000]; extern int num_threads; -extern char datapath_multi[NFILES][900]; +extern char datapath_multi[NFILES][900],datapath[900]; +extern int xaxis, yaxis; extern int ndatasets; +extern int grid_size_x; +extern int grid_size_y; +extern int num_w_planes; +extern int rank; +extern int size; +extern long nsectors; + +extern clock_t start, end, start0, startk, endk; +extern struct timespec begin, finish, begin0, begink, finishk; diff --git a/init.c b/init.c index 057c0818d240f2819f02b40c27e0d0deb79d7aba..5b5489385051a7086aaf558515667aec791d3aaf 100644 --- a/init.c +++ b/init.c @@ -4,9 +4,70 @@ #include "allvars.h" #include "proto.h" -void init() +void init(int argc, char * argv[]) { + // MESH SIZE + int local_grid_size_x;// = 8; + int local_grid_size_y;// = 8; + read_parameter_file(in.paramfile); + + if ( num_threads == 0 ) + { + fprintf(stderr, "Usage: %s number_of_OMP_Threads \n", num_threads); + exit(1); + } + clock_gettime(CLOCK_MONOTONIC, &begin0); + start0 = clock(); + // Intialize MPI environment + #ifdef USE_MPI + MPI_Init(&argc,&argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if(rank == 0)printf("Running with %d MPI tasks\n",size); + #ifdef USE_FFTW + fftw_mpi_init(); + #endif + #else + rank = 0; + size = 1; + #endif + + if(rank == 0)printf("Running with %d threads\n",num_threads); + + #ifdef ACCOMP + if(rank == 0){ + if (0 == omp_get_num_devices()) { + printf("No accelerator found ... exit\n"); + exit(255); + } + printf("Number of available GPUs %d\n", omp_get_num_devices()); + #ifdef NVIDIA + prtAccelInfo(); + #endif + } + #endif + + // set the local size of the image + local_grid_size_x = grid_size_x; + nsectors = NUM_OF_SECTORS; + if (nsectors < 0) nsectors = size; + local_grid_size_y = grid_size_y/nsectors; + //nsectors = size; + + // LOCAL grid size + xaxis = local_grid_size_x; + yaxis = local_grid_size_y; + + clock_gettime(CLOCK_MONOTONIC, &begin); + start = clock(); + + // INPUT FILES (only the first ndatasets entries are used) + strcpy(datapath,datapath_multi[0]); + + // Read metadata + fileName(datapath, in.metafile); + readMetaData(filename); } diff --git a/main.c b/main.c index 18bf8a28ec824068716d62ee839e131a22b4ea0a..f9bb61b3ba328c7c22ae0127c9b0ac2074f8a9a7 100644 --- a/main.c +++ b/main.c @@ -1,28 +1,7 @@ -#include -#include -#include -#ifdef USE_MPI -#include -#ifdef USE_FFTW -#include -#endif -#endif -#include -#include -#include -#include +#include #include "allvars.h" #include "proto.h" -#ifdef ACCOMP -#include "w-stacking_omp.h" -#else -#include "w-stacking.h" -#endif -#define PI 3.14159265359 -#define NUM_OF_SECTORS -1 -#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) -#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) -#define NOVERBOSE + // Linked List set-up struct sectorlist { @@ -41,10 +20,6 @@ void Push(struct sectorlist** headRef, long data) { // Main Code int main(int argc, char * argv[]) { - int rank; - int size; - - char datapath[900]; char srank[4]; @@ -57,51 +32,33 @@ int main(int argc, char * argv[]) double resolution; // MESH SIZE - int grid_size_x = 2048; - int grid_size_y = 2048; - int local_grid_size_x;// = 8; - int local_grid_size_y;// = 8; - int xaxis; - int yaxis; - int num_w_planes = 8; +// int local_grid_size_x;// = 8; +// int local_grid_size_y;// = 8; +// int xaxis; +// int yaxis; // DAV: the corresponding KernelLen is calculated within the wstack function. It can be anyway hardcoded for optimization int w_support = 7; - int num_threads;// = 4; double dx = 1.0/(double)grid_size_x; double dw = 1.0/(double)num_w_planes; double w_supporth = (double)((w_support-1)/2)*dx; - clock_t start, end, start0, startk, endk; - struct timespec begin, finish, begin0, begink, finishk; double elapsed; - long nsectors; - - if(argc > 1) - { - strcpy(in.paramfile,argv[1]); - printf("parameter file = %s\n", in.paramfile); - init(); - } - else - { - fprintf(stderr, "Parameter file is not given\n"); - exit(1); - } - +// long nsectors; - if ( num_threads == 0 ) - { - fprintf(stderr, "Wrong parameter: %s\n\n", argv[1]); - fprintf(stderr, "Usage: %s number_of_OMP_Threads \n", argv[0]); - exit(1); - } - - clock_gettime(CLOCK_MONOTONIC, &begin0); - start0 = clock(); - - // Intialize MPI environment -#ifdef USE_MPI + if(argc > 1) + { + strcpy(in.paramfile,argv[1]); + } + else + { + fprintf(stderr, "Parameter file is not given\n"); + exit(1); + } + + init(argc, argv); + /* + #ifdef USE_MPI MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); @@ -109,7 +66,6 @@ int main(int argc, char * argv[]) #ifdef USE_FFTW fftw_mpi_init(); #endif - MPI_Barrier(MPI_COMM_WORLD); #else rank = 0; size = 1; @@ -130,6 +86,11 @@ if(rank == 0){ } #endif + + + + + // set the local size of the image local_grid_size_x = grid_size_x; nsectors = NUM_OF_SECTORS; @@ -146,10 +107,11 @@ if(rank == 0){ // INPUT FILES (only the first ndatasets entries are used) strcpy(datapath,datapath_multi[0]); - // Read metadata + + // Read metadata fileName(datapath, in.metafile); readMetaData(filename); - +*/ // WATCH THIS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! int nsub = 1000; //int nsub = 10; @@ -171,8 +133,8 @@ if(rank == 0){ long nm_pe = (long)(metaData.Nmeasures/size); long remaining = metaData.Nmeasures%size; - long startrow = rank*nm_pe; - if (rank == size-1)nm_pe = nm_pe+remaining; + long startrow = rank*nm_pe; + if (rank == size-1)nm_pe = nm_pe+remaining; long Nmeasures_tot = metaData.Nmeasures; metaData.Nmeasures = nm_pe; diff --git a/proto.h b/proto.h index 644727f9be9ce6986b6b2191f9bc66cc4d6dd279..84ccbbd826264d59de55403ef149d0972f310e96 100644 --- a/proto.h +++ b/proto.h @@ -3,7 +3,7 @@ /* init.c */ -void init(); +void init(int argc, char * argv[]); void read_parameter_file(char *); void fileName(char datapath[900], char file[30]); void readMetaData(char fileLocal[1000]);