diff --git a/.gitignore b/.gitignore index b3ff3663bec2ef29f4d3c59f7c4678295622d002..e1937c268b697ca98d3fce12ce7e0cd65adf291b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ # Patterns .* *.png +*.yml # Folders tmp/ +tmp_*/ analysis_archive/ # Files @@ -15,6 +17,7 @@ rtasci/rtasci/cfg/leo_bkg.yml !rtamock/execute_run.sh !environment.yml !rtavis/rtavis/lib/*py +!astrort/configure/test.yml # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index cf4f9573e21ad541e58cdca690870036bf94604d..f06f20503dd8fa77a86c0381ea74712a8e9c2e34 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -20,12 +20,12 @@ class CheckConfiguration(): return self def check_tags(self): - tags = ['simulator', 'visibility'] + tags = ['simulator', 'visibility', 'logging'] assert self.conf.keys() == tags return self def check_simulator(self): - keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model'] + keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model', 'output'] assert self.conf['simulator'].keys() == keys assert type(self.conf['simulator']['name']) == str assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south'] @@ -36,6 +36,7 @@ class CheckConfiguration(): assert type(self.conf['simulator']['samples']) == int assert type(self.conf['simulator']['seed']) == int assert type(self.conf['simulator']['model']) == str + assert type(self.conf['simulator']['output']) == str return self def check_visibility(self): @@ -43,3 +44,36 @@ class CheckConfiguration(): assert self.conf['visibility'].keys() == keys assert type(self.conf['visibility']['start_time']) == str return self + + def check_logging(self): + keys = ['level', 'logfile', 'datfile'] + assert self.conf['logging'].keys() == keys + assert (type(self.conf['logging']['level']) == str or type(self.conf['logging']['level']) == int) + assert type(self.conf['logging']['logfile']) == str + assert type(self.conf['logging']['datfile']) == str + return self + + def check_slurm(self): + keys = ['nodes', 'tasks', 'cpus', 'mem', 'environment', 'name', 'account', 'partition'] + assert self.conf['slurm'].keys() == keys + assert type(self.conf['slurm']['nodes']) == int + assert type(self.conf['slurm']['tasks']) == int + assert type(self.conf['slurm']['cpus']) == int + assert type(self.conf['slurm']['mem']) == str + assert type(self.conf['slurm']['environment']) == str + assert type(self.conf['slurm']['name']) == str + assert type(self.conf['slurm']['account']) == str + assert type(self.conf['slurm']['partition']) == str + return self + + def check_mapper(self): + keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output'] + assert self.conf['mapper'].keys() == keys + assert type(self.conf['mapper']['exposure']) == int + assert type(self.conf['mapper']['smooth']) == (float or int) + assert type(self.conf['mapper']['pixelsize']) == (float or int) + assert type(self.conf['mapper']['center']) in ['pointing', 'source'] + assert type(self.conf['mapper']['plot']) == bool + assert type(self.conf['mapper']['region']) == bool + assert type(self.conf['mapper']['output']) == str + return self \ No newline at end of file diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..55fcc49d145bf7db2950bca77201278966d1a5a5 --- /dev/null +++ b/astrort/configure/logging.py @@ -0,0 +1,53 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import logging +from os import makedirs +from os.path import dirname, isfile, join + +def set_logger(level, filename=None): + log = logging.getLogger() + log.setLevel(level) + # console logger + formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s') + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + log.addHandler(consoleHandler) + # file logger + if filename is not None: + makedirs(dirname(filename), exist_ok=True) + if not isfile(filename): + f = open(filename, 'w+') + f.close() + fileHandler = logging.FileHandler(filename) + fileHandler.setFormatter(formatter) + log.addHandler(fileHandler) + return log + +def get_log_level(level): + if level in ['DEBUG', 'debug', 10]: + level = logging.DEBUG + elif level in ['INFO', 'info', 20]: + level = logging.INFO + elif level in ['WARN', 'WARNING', 'warn', 'warning', 30]: + level = logging.WARN + elif level in ['ERROR', 'error', 40]: + level = logging.ERROR + elif level in ['CRITICAL', 'critical', 50]: + level = logging.CRITICAL + else: + level = logging.NOTSET + return level + +def get_logfile(configuration, mode): + logfile = configuration['logging']['logfile'] + if mode not in logfile: + logfile = logfile.replace('.log', f'_{mode}.log') + if configuration[mode]['output'] not in logfile: + logfile = join(configuration[mode]['output'], logfile) + return logfile \ No newline at end of file diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py new file mode 100644 index 0000000000000000000000000000000000000000..deb8651bedad22650c775fc3dfd0bd360d4659c2 --- /dev/null +++ b/astrort/configure/slurmjobs.py @@ -0,0 +1,65 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +from yaml import dump +from os import system, makedirs +from os.path import join, dirname, abspath +from astrort.utils.wrap import load_yaml_conf +from astrort.configure.logging import set_logger, get_log_level + +def make_configuration(jobname_conf, configuration, node_number, mode): + # simulator + configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 + # logging + configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log') + configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat') + # write new configuration + with open(jobname_conf, 'w+') as f: + dump(configuration, f, default_flow_style=False) + +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): + # write sbatch + with open(jobname_sh, 'w+') as f: + f.write("#!/bin/bash") + #f.write("\n#SBATCH --nodes=1") + #f.write("\n#SBATCH --ntasks-per-node=1") + #f.write("\n#SBATCH --cpus-per-task=2") + f.write(f"\n#SBATCH --job-name={jobname}") + #f.write(f"\n#SBATCH --mem={slurmconf['memory']}") + f.write(f"\n#SBATCH --output={jobname_log}") + f.write(f"\n#SBATCH --account={slurmconf['account']}") + f.write(f"\n#SBATCH --partition={slurmconf['partition']}") + f.write(f"\n") + f.write(f"\nsource activate {slurmconf['environment']}") + if mode == 'simulator': + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") + elif mode == 'mapper': + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n") + else: + raise ValueError(f"Invalid 'mode' {mode}") + +def make_sbatch(jobname, configuration, node_number, mode): + output = configuration[mode]['output'] + jobname_sh = join(output, f"{jobname}_{mode}.sh") + jobname_log = join(output, f"{jobname}_{mode}.slurm") + jobname_conf = join(output, f"{jobname}_{mode}.yml") + make_configuration(jobname_conf, configuration, node_number, mode=mode) + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode) + system(f"sbatch {jobname_sh}") + +def slurm_submission(configuration_file, nodes, mode): + configuration = load_yaml_conf(configuration_file) + log = set_logger(get_log_level(configuration['logging']['level'])) + # create output dir + log.info(f"Creating {configuration[mode]['output']}") + makedirs(configuration[mode]['output'], exist_ok=True) + # sbatch jobs per each nodes + configuration['slurm']['nodes'] = nodes + for node_number in range(configuration['slurm']['nodes']): + jobname = f"{configuration['slurm']['name']}_{node_number+1}" + make_sbatch(jobname, configuration, node_number, mode=mode) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 5728707099cc9789caae62a95f69d6e9a478933e..645c04162c4bdced91a3e0402a67361da20debc5 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -1,15 +1,39 @@ simulator: - name: test + name: crab array: lst irf: North_z60_0.5h_LST - prod: prod5 + prod: prod5-v0.1 pointing: random - duration: 300 - samples: 5 + duration: 10 + samples: 2 seed: 1 model: $TEMPLATES$/crab.xml - output: /data01/homes/dipiano/astroRT/ + output: /data01/homes/dipiano/astroRT/astrort/testing/tmp +mapper: + exposure: 10 + smooth: 1 + pixelsize: 0.02 + center: pointing + plot: true + region: false + output: /data01/homes/dipiano/astroRT/astrort/testing/tmp + visibility: start_time: '2030-01-01T00:00:00' +logging: + level: CRITICAL + logfile: test.log + datfile: test.dat + +slurm: + nodes: 5 + tasks: 1 + cpus: 2 + mem: 32G + environment: astrort + name: test + account: dipiano + partition: large + diff --git a/astrort/scripts/merge_datfiles.py b/astrort/scripts/merge_datfiles.py new file mode 100644 index 0000000000000000000000000000000000000000..0f2a75edab8f6890dd51368a20a79e6c34d3b33b --- /dev/null +++ b/astrort/scripts/merge_datfiles.py @@ -0,0 +1,29 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +from os.path import join +from astrort.utils.wrap import load_yaml_conf, merge_data_info +from astrort.configure.logging import set_logger + +def main(configuration, mode): + configuration = load_yaml_conf(configuration) + logfile = join(configuration[mode]['output'], f'merge_{mode}_data.log') + log = set_logger(configuration['logging']['level'], logfile) + log.info(f"merge {mode} data files") + merge_data_info(configuration[mode], mode, log) + log.info(f"{mode} merge completed") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-m', '--mode', type=str, required=True, choices=['simulator', 'mapper'], help='Data table to merge') + args = parser.parse_args() + + main(args.configuration, args.mode) \ No newline at end of file diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..bebc4e24c2dbc6e964cbbe1300c74374d74e988c --- /dev/null +++ b/astrort/simulator/base_mapper.py @@ -0,0 +1,65 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +from time import time +from os import makedirs +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility, plot_map +from astrort.utils.utils import get_all_seeds +from astrort.configure.logging import set_logger, get_log_level, get_logfile +from astrort.configure.slurmjobs import slurm_submission + +def base_mapper(configuration_file, seeds=None): + clock = time() + configuration = load_yaml_conf(configuration_file) + logfile = get_logfile(configuration, mode='mapper') + datfile = logfile.replace('.log', '.dat') + # set logger + log = set_logger(get_log_level(configuration['logging']['level']), logfile) + # collect simulations to map + if seeds is None: + log.info(f"Mapping of all simulations found") + seeds = get_all_seeds(configuration['simulator']) + log.info(f"Mapper configured, took {time() - clock} s") + # create output dir + log.info(f"Output folder: {configuration['mapper']['output']}") + makedirs(configuration['mapper']['output'], exist_ok=True) + # start mapping + log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") + for seed in seeds: + clock_map = time() + # make map + fitsmap = execute_mapper_no_visibility(configuration, log) + log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s") + # make plot + if configuration['mapper']['plot']: + clock_plot = time() + plotmap = plot_map(fitsmap, log) + log.info(f"Plotting (seed = {seed}) complete, took {time() - clock_plot} s") + # timing simulation + clock_map = time() - clock_map + # save simulation data + write_mapping_info(configuration, datfile, clock_map) + configuration['simulator']['seed'] += 1 + # end simulations + log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n") + log.info(f"Process complete, took {time() - clock} s") + +def main(configuration, nodes): + if nodes == 0: + base_mapper(configuration) + else: + slurm_submission(configuration, nodes, mode='mapper') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + args = parser.parse_args() + + main(args.configuration, args.nodes) \ No newline at end of file diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 41672c6a027d5eaa51a2fc6cd2fd7ee1750a6b35..9a1878bc226426a85811e2855cd7467356b51f5f 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -7,16 +7,54 @@ # ***************************************************************************** import argparse -from rtasci.utils.RTACtoolsSimulation import RTACtoolsSimulation -from rtavis.utils.visibility import Visibility -from astrort.utils.wrap import load_yaml_conf -from astrort.cfg.check_configuration import CheckConfiguration +from time import time +from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing +from astrort.configure.logging import set_logger, get_log_level, get_logfile +from astrort.configure.slurmjobs import slurm_submission -parser = argparse.ArgumentParser(description='') -parser.add_argument('-cf', '--conf', type=str, required=True, help="Path of yaml configuration file") -args = parser.parse_args() +def base_simulator(configuration_file): + clock = time() + configuration = load_yaml_conf(configuration_file) + logfile = get_logfile(configuration, mode='simulator') + datfile = logfile.replace('.log', '.dat') + log = set_logger(get_log_level(configuration['logging']['level']), logfile) + log.info(f"Simulator configured, took {time() - clock} s") + # create output dir + log.info(f"Output folder: {configuration['simulator']['output']}") + #makedirs(configuration['simulator']['output'], exist_ok=True) + # start simulations + log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") + for i in range(configuration['simulator']['samples']): + clock_sim = time() + simulator = RTACtoolsSimulation() + # check pointing option + simulator, point = set_pointing(simulator, configuration['simulator'], log) + # complete configuration + simulator = configure_simulator_no_visibility(simulator, configuration['simulator'], log) + simulator.run_simulation() + log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete, took {time() - clock_sim} s") + # timing simulation + clock_sim = time() - clock_sim + # save simulation data + write_simulation_info(simulator, configuration['simulator'], point, datfile, clock_sim) + configuration['simulator']['seed'] += 1 + del simulator + # end simulations + log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") + log.info(f"Process complete, took {time() - clock} s") -conf = load_yaml_conf(args.conf) +def main(configuration, nodes): + if nodes == 0: + base_simulator(configuration) + else: + slurm_submission(configuration, nodes, mode='simulator') +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + args = parser.parse_args() + main(args.configuration, args.nodes) diff --git a/astrort/testing/test_astrort/test_utils/test_wrap.py b/astrort/simulator/stop_simulations.py similarity index 57% rename from astrort/testing/test_astrort/test_utils/test_wrap.py rename to astrort/simulator/stop_simulations.py index 04cb39b127a6c65a4572e9720bf6348b603b05a3..9c10a15bb91e54f4a36a9a070ca11105f2cfc357 100644 --- a/astrort/testing/test_astrort/test_utils/test_wrap.py +++ b/astrort/simulator/stop_simulations.py @@ -6,10 +6,12 @@ # Ambra Di Piano # ***************************************************************************** -import pytest -from astrort.utils.wrap import load_yaml_conf +import sys +from os import system + +start_job_id = int(sys.argv[1]) +jobs = int(sys.argv[2]) + +for i in range(jobs): + system(f"scancel {start_job_id+i}") -@pytest.mark.astrort_configuration -def test_load_yaml_conf(astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - assert type(configuration) == dict \ No newline at end of file diff --git a/astrort/templates/background.xml b/astrort/templates/background.xml new file mode 100644 index 0000000000000000000000000000000000000000..3cf3b742d6f80041c02ee008abed55b120ac9893 --- /dev/null +++ b/astrort/templates/background.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/astrort/templates/base_empty_map.fits b/astrort/templates/base_empty_map.fits new file mode 100644 index 0000000000000000000000000000000000000000..62ac4466c5e8b8648dacaa8288c82de9dad21090 Binary files /dev/null and b/astrort/templates/base_empty_map.fits differ diff --git a/astrort/templates/crab.xml b/astrort/templates/crab.xml index 47843b8dc4a3668259dae724a082fb609a48bb6f..bbeaed69f8fcf86af59ccd2ca57c874fa3bb727e 100644 --- a/astrort/templates/crab.xml +++ b/astrort/templates/crab.xml @@ -1,6 +1,6 @@ - + diff --git a/astrort/testing/conftest.py b/astrort/testing/conftest.py index 4edcb34153b41082da8212600473d4784e99d438..115a74fc7ce55bad00679d0f82b88fdacc659510 100644 --- a/astrort/testing/conftest.py +++ b/astrort/testing/conftest.py @@ -12,10 +12,10 @@ from os.path import join, dirname, abspath from os import makedirs @pytest.fixture(scope='function') -def astrort_configuration(): +def test_conf_file(): return join(dirname(abspath(astrort.__file__)), 'configure', 'test.yml') @pytest.fixture(scope='function') -def astrort_tmp_folder(): +def test_tmp_folder(): makedirs(join(dirname(abspath(astrort.__file__)), 'testing', 'tmp'), exist_ok=True) return join(dirname(abspath(astrort.__file__)), 'testing', 'tmp') diff --git a/astrort/testing/pytest.ini b/astrort/testing/pytest.ini index 5c0519734fd9fe1dace6d88900b93819bcf86a99..e79d0a6d5561f24b88b5d1bc77ee0f0656166266 100644 --- a/astrort/testing/pytest.ini +++ b/astrort/testing/pytest.ini @@ -1,3 +1,4 @@ [pytest] markers = - astrort_configuration: rtadeep test configurtion file \ No newline at end of file + test_conf_file: test configurtion file + test_tmp_folder: test tmp output folder \ No newline at end of file diff --git a/astrort/testing/test_astrort/test_configure/test_check_configuration.py b/astrort/testing/test_astrort/test_configure/test_check_configuration.py deleted file mode 100644 index 903a82db130f17a6360342963fcf715d29d00828..0000000000000000000000000000000000000000 --- a/astrort/testing/test_astrort/test_configure/test_check_configuration.py +++ /dev/null @@ -1,42 +0,0 @@ -# ***************************************************************************** -# Copyright (C) 2023 INAF -# This software is distributed under the terms of the BSD-3-Clause license -# -# Authors: -# Ambra Di Piano -# ***************************************************************************** - -import pytest -from astrort.utils.wrap import load_yaml_conf -from astrort.configure.check_configuration import CheckConfiguration - -@pytest.mark.astrort_configuration -class TestCheckConfiguration: - - def test_check(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check() - except AssertionError as e: - type(e) == AssertionError - - def test_check_tags(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_tags() - except AssertionError as e: - type(e) == AssertionError - - def test_check_simulator(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_simulator() - except AssertionError as e: - type(e) == AssertionError - - def test_check_visibility(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_visibility() - except AssertionError as e: - type(e) == AssertionError diff --git a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py b/astrort/testing/test_astrort/test_simulator/test_base_simulator.py deleted file mode 100644 index b20924049cf65b31b25d31aa33ad1cb078152307..0000000000000000000000000000000000000000 --- a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py +++ /dev/null @@ -1,22 +0,0 @@ -# ***************************************************************************** -# Copyright (C) 2023 INAF -# This software is distributed under the terms of the BSD-3-Clause license -# -# Authors: -# Ambra Di Piano -# ***************************************************************************** - -import pytest -from astrort.utils.wrap import load_yaml_conf -from astrort.cfg.check_configuration import CheckConfiguration - -@pytest.mark.rtadeep_configuration -class TestBaseSimulator: - - @pytest.mark.skip('to-do') - def test_base_simulator(self, rtadeep_configuration): - - # get configuration - configuration = load_yaml_conf(rtadeep_configuration) - - diff --git a/astrort/testing/test_astrort/test_utils/test_utils.py b/astrort/testing/test_astrort/test_utils/test_utils.py deleted file mode 100644 index a78255bafccb8b23112fb37f8440bed6a8515dd6..0000000000000000000000000000000000000000 --- a/astrort/testing/test_astrort/test_utils/test_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# ***************************************************************************** -# Copyright (C) 2023 INAF -# This software is distributed under the terms of the BSD-3-Clause license -# -# Authors: -# Ambra Di Piano -# ***************************************************************************** - -import pytest -from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov - -@pytest.mark.astrort_tmp_folder -@pytest.mark.parametrize('samples', [3, 5, 8, 10]) -def test_seeds_to_string_formatter(samples, astrort_tmp_folder): - name = seeds_to_string_formatter(samples, astrort_tmp_folder, name='test', seed=1) - - if samples <= 3: - assert name == f"{astrort_tmp_folder}/test_001" - elif samples <= 5: - assert name == f"{astrort_tmp_folder}/test_00001" - elif samples <= 8: - assert name == f"{astrort_tmp_folder}/test_00000001" - else: - assert name == f"{astrort_tmp_folder}/test_1" - -@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) -def test_get_instrument_fov(array): - fov = get_instrument_fov(array) - - if array == 'lst': - assert fov == 2.5 - elif array == 'mst': - assert fov == 3.5 - elif array == 'sst': - assert fov == 5 - else: - assert fov == 5 - return fov - - diff --git a/astrort/testing/test_configure/test_check_configuration.py b/astrort/testing/test_configure/test_check_configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..f31839b1089370cfce81a8665d0fe357c9cbb2c1 --- /dev/null +++ b/astrort/testing/test_configure/test_check_configuration.py @@ -0,0 +1,63 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from astrort.utils.wrap import load_yaml_conf +from astrort.configure.check_configuration import CheckConfiguration + +@pytest.mark.test_conf_file +class TestCheckConfiguration: + + def test_check(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check() + except AssertionError as e: + type(e) == AssertionError + + def test_check_tags(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_tags() + except AssertionError as e: + type(e) == AssertionError + + def test_check_simulator(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_simulator() + except AssertionError as e: + type(e) == AssertionError + + def test_check_visibility(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_visibility() + except AssertionError as e: + type(e) == AssertionError + + def test_check_logging(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_logging() + except AssertionError as e: + type(e) == AssertionError + + def test_check_slurm(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_logging() + except AssertionError as e: + type(e) == AssertionError + + def test_check_mapper(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_mapper() + except AssertionError as e: + type(e) == AssertionError \ No newline at end of file diff --git a/astrort/testing/test_configure/test_logging.py b/astrort/testing/test_configure/test_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..8567b8296aa9752a50bc432b7e12e0df7703f449 --- /dev/null +++ b/astrort/testing/test_configure/test_logging.py @@ -0,0 +1,36 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import logging +from os.path import join, isfile +from astrort.configure.logging import * +from astrort.utils.wrap import load_yaml_conf + +def test_set_logger(): + assert get_log_level('DEBUG') == logging.DEBUG + assert get_log_level('INFO') == logging.INFO + assert get_log_level('WARNING') == logging.WARNING + assert get_log_level('ERROR') == logging.ERROR + assert get_log_level('CRITICAL') == logging.CRITICAL + +@pytest.mark.test_tmp_folder +def test_set_logger(test_tmp_folder): + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) + log.debug('TEST') + assert isfile(join(test_tmp_folder, 'test_set_logger.log')) is True + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_get_logfile(test_conf_file, mode): + conf = load_yaml_conf(test_conf_file) + + logfile = get_logfile(conf, mode) + assert mode in logfile + + diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py new file mode 100644 index 0000000000000000000000000000000000000000..dc6ccb17ebc52dfa599077180a7a008bb66918fe --- /dev/null +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -0,0 +1,60 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from shutil import rmtree +from os import listdir, makedirs +from os.path import isfile, join +from astrort.configure.slurmjobs import make_configuration, make_sh +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_make_configuration(test_conf_file, mode): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf[mode]['output']) + makedirs(conf[mode]['output'], exist_ok=True) + + # make configurations + for node_number in range(conf['slurm']['nodes']): + node_number += 1 + jobname = f"{conf['slurm']['name']}_{node_number}" + jobname_conf = join(conf[mode]['output'], f"job_{jobname}_{mode}.yml") + make_configuration(jobname_conf, conf, node_number, mode) + + # check output + expected_configurations = conf['slurm']['nodes'] + found_configurations = len([f for f in listdir(conf[mode]['output']) if isfile(join(conf[mode]['output'], f)) and '.yml' in f and conf['slurm']['name'] in f]) + assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_make_sh(test_conf_file, mode): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf[mode]['output']) + makedirs(conf[mode]['output'], exist_ok=True) + + # make configurations + output = conf[mode]['output'] + for node_number in range(conf['slurm']['nodes']): + node_number += 1 + jobname = f"{conf['slurm']['name']}_{node_number}" + jobname_sh = join(output, f"job_{jobname}_{mode}.sh") + jobname_log = join(output, f"job_{jobname}_{mode}.log") + jobname_conf = join(output, f"job_{jobname}_{mode}.yml") + make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log, mode=mode) + + # check output + expected_sh = conf['slurm']['nodes'] + found_sh = len([f for f in listdir(conf[mode]['output']) if isfile(join(conf[mode]['output'], f)) and '.sh' in f and conf['slurm']['name'] in f and mode in f]) + assert found_sh == expected_sh, f"Expected {expected_sh} files for {mode}, found {found_sh}" + diff --git a/astrort/testing/test_simulator/test_base_mapper.py b/astrort/testing/test_simulator/test_base_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..f0df4542de02ec0b384c44d7aa95a4f1a778c61a --- /dev/null +++ b/astrort/testing/test_simulator/test_base_mapper.py @@ -0,0 +1,37 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import numpy as np +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator +from astrort.simulator.base_mapper import base_mapper +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('seeds', [None, list()]) +def test_base_mapper(test_conf_file, seeds): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['mapper']['output'], ignore_errors=True) + + # run simulator + base_simulator(test_conf_file) + if type(seeds) == list: + seeds = np.arange(conf['simulator']['samples']) + base_mapper(test_conf_file, seeds) + + # check output + expected_maps = conf['simulator']['samples'] + found_maps = len([f for f in listdir(conf['mapper']['output']) if isfile(join(conf['mapper']['output'], f)) and '.fits' in f and conf['simulator']['name'] in f and 'map' in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + + diff --git a/astrort/testing/test_simulator/test_base_simulator.py b/astrort/testing/test_simulator/test_base_simulator.py new file mode 100644 index 0000000000000000000000000000000000000000..fc716e7b9524eab8f0d64da42c22b1e527a3b468 --- /dev/null +++ b/astrort/testing/test_simulator/test_base_simulator.py @@ -0,0 +1,32 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +def test_base_simulator(test_conf_file): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['simulator']['output'], ignore_errors=True) + + # run simulator + base_simulator(test_conf_file) + + # check output + expected_simulations = conf['simulator']['samples'] + found_simulations = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.fits' in f and conf['simulator']['name'] in f]) + assert found_simulations == expected_simulations, f"Expected {expected_simulations} simulations, found {found_simulations}" + + + diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8187a478f661c977bf268b501601385b33ec8350 --- /dev/null +++ b/astrort/testing/test_utils/test_utils.py @@ -0,0 +1,96 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import numpy as np +from os.path import isdir +from astrort.utils.utils import * +from astrort.utils.wrap import * + +@pytest.mark.test_tmp_folder +@pytest.mark.parametrize('samples', [3, 5, 8, 10]) +def test_seeds_to_string_formatter_files(samples, test_tmp_folder): + name = seeds_to_string_formatter_files(samples, test_tmp_folder, name='test', seed=1, ext='fits') + + if samples <= 1e3: + assert name == f"{test_tmp_folder}/test_001.fits" + elif samples <= 1e5: + assert name == f"{test_tmp_folder}/test_00001.fits" + elif samples <= 1e8: + assert name == f"{test_tmp_folder}/test_00000001.fits" + else: + assert name == f"{test_tmp_folder}/test_1.fits" + +@pytest.mark.parametrize('samples', [3, 5, 8, 10]) +def test_seeds_to_string_formatter(samples): + name = seeds_to_string_formatter(samples, name='test', seed=1) + + if samples <= 1e3: + assert name == f"test_001" + elif samples <= 1e5: + assert name == f"test_00001" + elif samples <= 1e8: + assert name == f"test_00000001" + else: + assert name == f"test_1" + +@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) +def test_get_instrument_fov(array): + fov = get_instrument_fov(array) + + if array == 'lst': + assert fov == 2.5 + elif array == 'mst': + assert fov == 3.5 + elif array == 'sst': + assert fov == 5 + else: + assert fov == 5 + +@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) +def test_get_instrument_tev_range(array): + erange = get_instrument_tev_range(array) + + if array == 'lst': + assert erange == [0.03, 5] + elif array == 'mst': + assert erange == [1, 50] + elif array == 'sst': + assert erange == [5, 150] + else: + assert erange == [0.03, 150] + +@pytest.mark.parametrize('zenith', ['irf_z60', 'irf_z40', 'irf_z20']) +def test_adjust_tev_range_to_irf(zenith): + erange = adjust_tev_range_to_irf([0.03, 150], zenith) + + if 'z60' in zenith: + assert erange[0] == 0.11 + elif 'z60' in zenith: + assert erange[0] == 0.04 + elif 'z20' in zenith: + assert erange[0] == 0.03 + +@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'north', 'south']) +def test_select_irf(array): + prod = 'prod5-v0.1' + path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') + irf = select_random_irf(array, prod) + + assert array in irf.lower() + assert 'share/caldb/data/cta' in join(path, irf).lower() + assert isdir(join(path, irf)) is True + +@pytest.mark.test_conf_file +def test_get_all_seeds(test_conf_file): + conf = load_yaml_conf(test_conf_file) + seeds = get_all_seeds(conf['simulator']) + assert seeds.all() == np.array([1, 2]).all() + +def test_map_template(): + assert map_template() == join(dirname(abspath(__file__)).replace('testing/test_utils', 'templates'), 'base_empty_map.fits') \ No newline at end of file diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a9e8afb3ac3fa4b338a799fd09d90ded503fe2 --- /dev/null +++ b/astrort/testing/test_utils/test_wrap.py @@ -0,0 +1,166 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import logging +import numpy as np +from shutil import rmtree +from astrort.utils.wrap import * +from astrort.configure.logging import set_logger +from astrort.simulator.base_simulator import base_simulator +from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation + +@pytest.mark.test_conf_file +def test_load_yaml_conf(test_conf_file): + configuration = load_yaml_conf(test_conf_file) + assert type(configuration) == dict + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('model', ['crab.xml', 'background.xml']) +def test_randomise_pointing_sim(test_conf_file, model): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['name'] = model.replace('.xml', '') + conf['simulator']['model'] = f'$TEMPLATES$/{model}' + pointing = randomise_pointing_sim(conf['simulator']) + assert type(pointing) == dict + for key in pointing.keys(): + assert key in ['point_ra', 'point_dec', 'offset', 'source_ra', 'source_dec'] + assert type(pointing['point_ra']) == type(np.float64(1)) + assert type(pointing['point_dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) + assert type(pointing['source_ra']) == type(np.float64(1)) + assert type(pointing['source_dec']) == type(np.float64(1)) + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('model', ['crab.xml', 'background.xml']) +def test_get_point_source_info(test_conf_file, model): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['name'] = model.replace('.xml', '') + conf['simulator']['model'] = f'$TEMPLATES$/{model}' + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + assert type(pointing) == dict + for key in pointing.keys(): + assert key in ['point_ra', 'point_dec', 'offset', 'source_ra', 'source_dec'] + if 'background.xml' not in model: + assert type(pointing['point_ra']) == type(np.float64(1)) + assert type(pointing['point_dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) + assert type(pointing['source_ra']) == type(np.float64(1)) + assert type(pointing['source_dec']) == type(np.float64(1)) + else: + assert type(pointing['point_ra']) == type(1) + assert type(pointing['point_dec']) == type(1) + assert type(pointing['offset']) == type(np.nan) + assert type(pointing['source_ra']) == type(np.nan) + assert type(pointing['source_dec']) == type(np.nan) + +@pytest.mark.test_conf_file +def test_write_simulation_info(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + datfile = join(conf['simulator']['output'], 'simulator.dat') + sim = RTACtoolsSimulation() + clock = 1 + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + del sim + +@pytest.mark.test_tmp_folder +@pytest.mark.test_conf_file +def test_merge_simulation_info(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + sim = RTACtoolsSimulation() + clock = 1 + for i in range(5): + sim.seed = i + datfile = join(conf['simulator']['output'], f'job_{i}_simulator.dat') + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) + merge_simulation_info(conf['simulator'], log) + assert isfile(join(conf['simulator']['output'], 'merged_sim_data.dat')) + del sim + +@pytest.mark.test_tmp_folder +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_merge_data_info(test_conf_file, test_tmp_folder, mode): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + sim = RTACtoolsSimulation() + clock = 1 + for i in range(5): + sim.seed = i + datfile = join(conf['simulator']['output'], f'job_{i}_simulator.dat') + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + + datfile = join(conf['mapper']['output'], f'job_{i}_mapper.dat') + write_mapping_info(conf, datfile, clock) + assert isfile(datfile) + + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) + merge_data_info(conf[mode], mode, log) + assert isfile(join(conf[mode]['output'], f'merged_{mode}_data.dat')) + del sim + +@pytest.mark.test_conf_file +def test_write_mapping_info(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + datfile = join(conf['mapper']['output'], 'mapper.dat') + clock = 1 + write_mapping_info(conf, datfile, clock) + assert isfile(datfile) + +@pytest.mark.test_conf_file +def test_configure_simulator_no_visibility(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + log = set_logger(logging.CRITICAL) + sim = RTACtoolsSimulation() + sim = configure_simulator_no_visibility(sim, conf['simulator'], log) + + assert sim.t == [0, conf['simulator']['duration']] + assert sim.seed == conf['simulator']['seed'] + assert sim.set_log is False + assert sim.caldb == conf['simulator']['prod'] + del sim + +@pytest.mark.test_conf_file +def test_execute_mapper_no_visibility(test_conf_file): + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['mapper']['output'], ignore_errors=True) + conf['simulator']['samples'] = 1 + log = set_logger(logging.CRITICAL) + + # run simulator + base_simulator(test_conf_file) + fitsmap = execute_mapper_no_visibility(conf, log) + assert isfile(fitsmap) + +@pytest.mark.test_conf_file +def test_plot_map(test_conf_file): + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['mapper']['output'], ignore_errors=True) + conf['simulator']['samples'] = 1 + log = set_logger(logging.CRITICAL) + + # run simulator + base_simulator(test_conf_file) + fitsmap = execute_mapper_no_visibility(conf, log) + plotmap = plot_map(fitsmap, log) + assert isfile(plotmap) \ No newline at end of file diff --git a/astrort/utils/mapping.py b/astrort/utils/mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..4ce7c150c5f32e64245b7f6ed5d6401a2bab9bee --- /dev/null +++ b/astrort/utils/mapping.py @@ -0,0 +1,226 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import shutil +import numpy as np +from astropy.io import fits +from scipy.ndimage import gaussian_filter + +class Mapper(): + '''Class that read, writes and manages the FITS data format.''' + def __init__(self, logger) -> None: + self.hdr_fits_keys = ['NAXIS1', 'NAXIS2', 'CRPIX1', 'CRPIX2', 'CDELT1', 'CDELT2', 'CRVAL1', 'CRVAL2', 'DATE-OBS', 'TIME-OBS', 'DATE-END', 'TIME-END', 'TELAPSE', 'ONTIME', 'LIVETIME', 'EXPOSURE', 'DEADC', 'E_MIN', 'E_MAX'] + + self.set_logger(logger=logger) + pass + + def set_logger(self, logger): + self.log = logger + return self + + def empty_header(self, hdr): + hdr['NAXIS1'] = 0 + hdr['NAXIS2'] = 0 + hdr['CRPIX1'] = 0.0 + hdr['CRPIX2'] = 0.0 + hdr['CDELT1'] = 0.0 + hdr['CDELT2'] = 0.0 + hdr['CRVAL1'] = 0.0 + hdr['CRVAL2'] = 0.0 + #hdr['LONPOLE'] = 0.0 + #hdr['LATPOLE'] = 0.0 + hdr['CREATOR'] = 'RTAPH' + hdr['DATE-OBS'] = '2000-01-01' + hdr['TIME-OBS'] = '00:00:00' + hdr['DATE-END'] = '2000-01-01' + hdr['TIME-END'] = '00:00:00' + hdr['TELAPSE'] = 0.0 + hdr['ONTIME'] = 0.0 + hdr['LIVETIME'] = 0.0 + hdr['EXPOSURE'] = 0.0 + hdr['DEADC'] = 0.0 + hdr['E_MIN'] = 0.0 + hdr['E_MAX'] = 0.0 + return hdr + + def get_template(self): + '''TODO''' + return + + def convert_countmap_in_template(self, skymap, template): + shutil.copy(skymap, template) + with fits.open(template, mode='update') as h: + h['SKYMAP'].data = np.empty_like(h['SKYMAP'].data) + h['SKYMAP'].header = self.empty_header(h['SKYMAP'].header) + h.flush() + return self + + def update_countmap_hdr(self, hdr, hdr_info): + hdr['NAXIS1'] = hdr_info['nbins'] + hdr['NAXIS2'] = hdr_info['nbins'] + hdr['CRPIX1'] = hdr_info['xref'] + hdr['CRPIX2'] = hdr_info['yref'] + hdr['CDELT1'] = hdr_info['pixelsize'] + hdr['CDELT2'] = hdr_info['pixelsize'] + hdr['CRVAL1'] = hdr_info['point_ra'] + hdr['CRVAL2'] = hdr_info['point_dec'] + #hdr['LONPOLE'] = hdr_info['lonpole'] + #hdr['LATPOLE'] = hdr_info['latpole'] + hdr['CREATOR'] = 'RTAPH' + hdr['DATE-OBS'] = hdr_info['date_obs'] + hdr['TIME-OBS'] = hdr_info['time_obs'] + hdr['DATE-END'] = hdr_info['date_end'] + hdr['TIME-END'] = hdr_info['time_end'] + hdr['TELAPSE'] = hdr_info['telapse'] + hdr['ONTIME'] = hdr_info['ontime'] + hdr['LIVETIME'] = hdr_info['livetime'] + hdr['EXPOSURE'] = hdr_info['exposure'] + hdr['DEADC'] = hdr_info['deadc'] + hdr['E_MIN'] = hdr_info['emin'] + hdr['E_MAX'] = hdr_info['emax'] + return hdr + + def get_header_info_from_dl3(self, hdr): + hdr_info = {} + for k in self.hdr_fits_keys: + hdr_info[k] = hdr[k] + return hdr_info + + def get_dl3_hdr(self, dl3_file): + with fits.open(dl3_file) as h: + hdr = h['EVENTS'].header + return hdr + + def set_dl3_hdr(self, dl3_file, hdr_fits): + with fits.open(dl3_file, mode='update') as h: + for k in hdr_fits.keys(): + h['EVENTS'].header[k] = hdr_fits[k] + h.flush() + return self + + def get_dl3_data(self, dl3_file): + with fits.open(dl3_file) as h: + data = h['EVENTS'].data + return data + + def set_dl3_data(self, dl3_file, data, GTI=None): + with fits.open(dl3_file, mode='update') as h: + h['EVENTS'].data = data + if GTI is not None: + h['GTI'].data[0] = GTI + h.flush() + return self + + def get_dl4_hdr(self, dl4_file): + with fits.open(dl4_file) as h: + hdr = h['SKYMAP'].header + return hdr + + def set_dl4_hdr(self, dl4_file, hdr_fits): + with fits.open(dl4_file, mode='update') as h: + for k in hdr_fits.keys(): + h['SKYMAP'].header[k] = hdr_fits[k] + h.flush() + return self + + def get_dl4_data(self, dl4_file): + with fits.open(dl4_file) as h: + data = h['SKYMAP'].data + return data + + def set_dl4_data(self, dl4_file, data): + with fits.open(dl4_file, mode='update') as h: + h['SKYMAP'].data = data + h.flush() + return self + + def get_binning_size(self, maproi=5, pixelsize=0.02): + nbins = int(maproi*2/pixelsize) + return nbins + + def get_pixel_reference_point(self, nbins): + return nbins/2+0.5, nbins/2+0.5 + + def store_hdr_in_dict(self, dl3_hdr, maproi=5, pixelsize=0.02): + hdr_fits = {} + nbins = self.get_binning_size(maproi=maproi, pixelsize=pixelsize) + hdr_fits['NAXIS1'] = nbins + hdr_fits['NAXIS2'] = nbins + xref, yref = self.get_pixel_reference_point(nbins=nbins) + hdr_fits['CRPIX1'] = xref + hdr_fits['CRPIX2'] = yref + hdr_fits['CDELT1'] = pixelsize + hdr_fits['CDELT2'] = pixelsize + hdr_fits['CRVAL1'] = float(dl3_hdr['RA_PNT']) + hdr_fits['CRVAL2'] = float(dl3_hdr['DEC_PNT']) + hdr_fits['CREATOR'] = 'ASTRORT' + hdr_fits['TSTART'] = float(dl3_hdr['TSTART']) + hdr_fits['TSTOP'] = float(dl3_hdr['TSTOP']) + hdr_fits['TELAPSE'] = dl3_hdr['TELAPSE'] + hdr_fits['ONTIME'] = dl3_hdr['ONTIME'] + hdr_fits['LIVETIME'] = dl3_hdr['LIVETIME'] + hdr_fits['EXPOSURE'] = dl3_hdr['LIVETIME'] + hdr_fits['DEADC'] = dl3_hdr['DEADC'] + erange = dl3_hdr['DSVAL2'].split(':') + hdr_fits['E_MIN'] = float(erange[0]) + hdr_fits['E_MAX'] = float(erange[1]) + hdr_fits['RADESYS'] = dl3_hdr['RADECSYS'] + return hdr_fits + + def get_countmap_in_fits(self, dl3_file, template, pixelsize=0.02, maproi=5, trange=None, erange=None, sigma=1, fitsname='skymap.fits'): + shutil.copy(template, fitsname) + dl3_hdr = self.get_dl3_hdr(dl3_file=dl3_file) + pointing = {'ra': float(dl3_hdr['RA_PNT']), 'dec': float(dl3_hdr['DEC_PNT'])} + hdr_fits = self.store_hdr_in_dict(dl3_hdr=dl3_hdr, maproi=maproi, pixelsize=pixelsize) + dl3_data = self.get_dl3_data(dl3_file=dl3_file) + dl3_data = self.selection_cuts(dl3_data=dl3_data, pointing=pointing, trange=trange, erange=erange, maproi=maproi) + if sigma != 0: + dl4_data = self.from_dl3_to_dl4(dl3_data=dl3_data, pointing=pointing, maproi=maproi, pixelsize=0.02, sigma=sigma) + else: + dl4_data = self.from_dl3_to_dl4(dl3_data=dl3_data, pointing=pointing, maproi=maproi, pixelsize=pixelsize) + self.set_dl4_hdr(dl4_file=fitsname, hdr_fits=hdr_fits) + self.set_dl4_data(dl4_file=fitsname, data=dl4_data) + return + + def get_extent(self, pointing, roi): + extent = [pointing['ra']-roi, pointing['ra']+roi, pointing['dec']-roi, pointing['dec']+roi] + return extent + + def heatmap_with_smoothing(self, x, y, extent, sigma=1, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + heatmap = gaussian_filter(heatmap, sigma=sigma) + return heatmap.T + + def heatmap(self, x, y, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + return heatmap.T + + def from_dl3_to_dl4(self, dl3_data, pointing, maproi=5, pixelsize=0.02, sigma=1): + ra = np.array(dl3_data.field('RA')).flatten() + dec = np.array(dl3_data.field('DEC')).flatten() + nbins = self.get_binning_size(maproi=maproi, pixelsize=pixelsize) + extent = self.get_extent(pointing=pointing, roi=maproi) + if sigma != 0: + dl4_data = self.heatmap_with_smoothing(ra, dec, extent=extent, bins=nbins, sigma=sigma) + else: + dl4_data = self.heatmap(ra, dec, extent=extent, bins=nbins) + return dl4_data + + def selection_cuts(self, dl3_data, pointing, trange=None, erange=None, maproi=None): + if trange != None: + dl3_data = dl3_data[(dl3_data['TIME'] > trange[0]) & (dl3_data['TIME'] < trange[1])] + if erange != None: + dl3_data = dl3_data[(dl3_data['ENERGY'] > erange[0]) & (dl3_data['ENERGY'] < erange[1])] + if maproi != None: + dl3_data = dl3_data[(np.abs(np.abs(dl3_data['RA'])-np.abs(pointing['ra'])) < maproi)] + dl3_data = dl3_data[(np.abs(np.abs(dl3_data['DEC'])-np.abs(pointing['dec'])) < maproi)] + if len(dl3_data) == 0: + self.log.warning("Empty photon list selection.") + return dl3_data diff --git a/astrort/utils/plotting.py b/astrort/utils/plotting.py new file mode 100644 index 0000000000000000000000000000000000000000..3d12d46feb811558b771af0f36bbb0a7468ed575 --- /dev/null +++ b/astrort/utils/plotting.py @@ -0,0 +1,244 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import numpy as np +import matplotlib as mpl +mpl.use('Agg') +import matplotlib.pyplot as plt +from astropy.io import fits +from scipy.ndimage import gaussian_filter +from astropy.wcs import WCS +from matplotlib.colors import SymLogNorm +from astropy import units as u + +class Plotter(): + def __init__(self, logger) -> None: + self.set_logger(logger=logger) + pass + + def set_logger(self, logger): + self.log = logger + return self + + def set_target(self, ra=83.63, dec=22.01, rad=0.2, color='white'): + self.target = {'ra': ra, 'dec': dec, 'rad': rad, 'color': color} + return self + + def set_target_from_dict(self, target, rad=0.2, color='white'): + self.target = target + self.target['rad'] = rad + self.target['color'] = color + return self + + def set_pointing(self, ra=83.63, dec=22.51): + self.pointing = {'ra': ra, 'dec': dec, 'ftm': 'k+'} + return self + + def set_pointing_from_dict(self, pointing): + self.pointing = pointing + return self + + def __check_target(self): + assert isinstance(self.target, dict) is True, 'check that you have set the target correctly.' + return self + + def __check_pointing(self): + assert isinstance(self.pointing, dict) is True, 'check that you have set the pointing correctly.' + return self + + def set_path(self, path): + self.path = path + return self + + def load_fits(self, filename, extension='SKYMAP'): + h = fits.open(filename) + h.info() + data = h[extension].data + return data + + def heatmap_with_smoothing(self, x, y, sigma, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + heatmap = gaussian_filter(heatmap, sigma=sigma) + return heatmap.T, extent + + def heatmap(self, x, y, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + return heatmap.T, extent + + def get_extent(self, roi): + extent = [self.pointing['ra']-roi, self.pointing['ra']+roi, self.pointing['dec']-roi, self.pointing['dec']+roi] + return extent + + def set_wcs(self, point_ref, pixelsize): + w = WCS(naxis=2) + w.wcs.ctype = ['RA---CAR', 'DEC--CAR'] + w.wcs.crpix = [point_ref, point_ref] + w.wcs.crval = [self.pointing['ra'], self.pointing['dec']] + w.wcs.cdelt = [-pixelsize, pixelsize] + w.wcs.lonpole = 0.0 + w.wcs.latpole = 67.49 + return w + + def reshape_coords(self, ra, dec): + assert np.shape(ra) == np.shape(dec), 'RA and DEC shapes do not match' + coords = np.array([ra, dec]) + coords = coords.reshape((len(ra), 2)) + return coords + + def reshape_ra_dec(self, coords): + ra, dec = [], [] + for i in range(len(coords)): + ra.append(coords[i][0]) + dec.append(coords[i][1]) + ra = np.array(ra) + dec = np.array(dec) + return ra, dec + + def counts_map(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): + extent = self.get_extent(roi) + # counts map + data = self.get_phlist_data(file=file) + # select counts map + data = self.phlist_selection_cuts(data=data, trange=trange, erange=erange, roi=roi) + # axis and binning + ra = np.array(data.field('RA')).flatten() + dec = np.array(data.field('DEC')).flatten() + bins = int(roi*2/pixelsize) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot() + if sigma != 0: + hist = self.heatmap_with_smoothing(ra, dec, sigma=sigma, extent=extent, bins=bins) + else: + hist = self.heatmap(ra, dec, extent=extent, bins=bins) + img = ax.imshow(hist[0], norm=SymLogNorm(1, base=10), origin='lower', extent=extent, cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + # pointing + if isinstance(self.pointing, dict): + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + ax.set_xlim((extent[0], extent[1])) + ax.set_ylim((extent[2], extent[3])) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def counts_map_with_wcs(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): + extent = self.get_extent(roi=roi) + # counts map + data = self.get_phlist_data(file=file) + # select counts map + data = self.phlist_selection_cuts(data=data, trange=trange, erange=erange, roi=roi) + # axis and binning + ra = np.array(data.field('RA')).flatten() + dec = np.array(data.field('DEC')).flatten() + bins = int(roi*2/pixelsize) + # wcs + wcs = self.set_wcs(point_ref=bins/2+0.5, pixelsize=pixelsize) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot(projection=wcs) + if sigma != 0: + hist = self.heatmap_with_smoothing(ra, dec, sigma=sigma, extent=extent, bins=bins) + else: + hist = self.heatmap(ra, dec, extent=extent, bins=bins) + ax.coords[0].set_format_unit(u.deg) + ax.coords[1].set_format_unit(u.deg) + img = ax.imshow(hist[0], norm=SymLogNorm(1, base=10), interpolation='gaussian', extent=extent, cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + # pointing + if isinstance(self.pointing, dict): + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def plot_fits_skymap(self, file, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', logbar=False): + + # get map + data, wcs = self.get_skymap_data_with_wcs(file=file) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot(projection=wcs) + ax.coords[0].set_format_unit(u.deg) + ax.coords[1].set_format_unit(u.deg) + if logbar: + img = plt.imshow(data, norm=SymLogNorm(1, base=10), origin='lower', interpolation='gaussian', cmap=cmap) + else: + img = plt.imshow(data, interpolation='gaussian', cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def get_skymap_data_with_wcs(self, file): + with fits.open(file) as h: + try: + w = WCS(h['SKYMAP'].header) + data = h['SKYMAP'].data + except KeyError as e: + self.log.error(f'Missing "SKYMAP" extention, the input file may not be a compatible counts map. {e}') + raise KeyError(f'Missing "SKYMAP" extention, the input file may not be a compatible counts map. {e}') + return data, w + + def get_phlist_data(self, file): + with fits.open(file) as h: + try: + data = h['EVENTS'].data + except KeyError as e: + self.log.error(f'Missin "EVENTS" extention, the input file may not be a compatible photon list. {e}') + raise KeyError(f'Missin "EVENTS" extention, the input file may not be a compatible photon list. {e}') + if len(data) == 0: + self.log.warning("Empty photon list.") + return data + + def phlist_selection_cuts(self, data, trange=None, erange=None, roi=None): + if trange != None: + data = data[(data['TIME'] > trange[0]) & (data['TIME'] < trange[1])] + if erange != None: + data = data[(data['ENERGY'] > erange[0]) & (data['ENERGY'] < erange[1])] + if roi != None: + data = data[(np.abs(np.abs(data['RA'])-np.abs(self.pointing['ra'])) < roi)] + data = data[(np.abs(np.abs(data['DEC'])-np.abs(self.pointing['dec'])) < roi)] + if len(data) == 0: + self.log.warning("Empty photon list selection.") + return data diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index c6723b0b8d78751b03f7b0af5365589d3610c724..4b3a8d99eec0830159f77011ae5c64a09af01db3 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -6,17 +6,37 @@ # Ambra Di Piano # ***************************************************************************** -from os.path import join - -def seeds_to_string_formatter(samples, output, name, seed): - if samples <= 3: - name = join(output, f"{name}_{seed:03d}") - elif samples <= 5: - name = join(output, f"{name}_{seed:05d}") - elif samples <= 8: - name = join(output, f"{name}_{seed:08d}") +import random +import numpy as np +from os import listdir +from os.path import join, expandvars, dirname, abspath + +def map_template(): + return join(dirname(abspath(__file__)).replace('utils', 'templates'), 'base_empty_map.fits') + +def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=None): + if samples <= 1e3: + name = join(output, f"{name}_{seed:03d}.{ext}") + elif samples <= 1e5: + name = join(output, f"{name}_{seed:05d}.{ext}") + elif samples <= 1e8: + name = join(output, f"{name}_{seed:08d}.{ext}") + else: + name = join(output, f"{name}_{seed}.{ext}") + # suffix + if suffix is not None: + name.replace(f'.{ext}', f'_{suffix}.{ext}') + return name + +def seeds_to_string_formatter(samples, name, seed): + if samples <= 1e3: + name = join(f"{name}_{seed:03d}") + elif samples <= 1e5: + name = join(f"{name}_{seed:05d}") + elif samples <= 1e8: + name = join(f"{name}_{seed:08d}") else: - name = join(output, f"{name}_{seed}") + name = join(f"{name}_{seed}") return name def get_instrument_fov(instrument): @@ -28,4 +48,37 @@ def get_instrument_fov(instrument): fov = 5 else: fov = 5 - return fov \ No newline at end of file + return fov + +def get_instrument_tev_range(array): + if array == 'lst': + erange = [0.03, 5] + elif array == 'mst': + erange = [1, 50] + elif array == 'sst': + erange = [5, 150] + else: + erange = [0.03, 150] + return erange + +def adjust_tev_range_to_irf(erange, irf): + # minimum energy + if "z60" in irf and erange[0] < 0.11: + erange[0] = 0.11 + elif "z40" in irf and erange[0] < 0.04: + erange[0] = 0.04 + elif "z20" in irf and erange[0] < 0.03: + erange[0] = 0.03 + return erange + +def select_random_irf(array, prod): + path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') + irfs = listdir(path) + irf = random.choice([i for i in irfs if array in i.lower()]) + return irf + +def get_all_seeds(simulator): + start_seed = simulator['seed'] + samples = simulator['samples'] + seeds = np.arange(start_seed, samples+start_seed, step=1) + return seeds \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 807fc6f7f48c640f3f0e0bf1203c4c6984db7b1e..37876d698322313946a9c16ccd8d0ffe4700cc2d 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -7,8 +7,16 @@ # ***************************************************************************** import yaml -from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov +import numpy as np +import pandas as pd +import astropy.units as u +from os.path import dirname, abspath, join, basename, isfile +from rtasci.lib.RTAManageXml import ManageXml +from astropy.coordinates import SkyCoord +from astrort.utils.utils import * from astrort.configure.check_configuration import CheckConfiguration +from astrort.utils.mapping import Mapper +from astrort.utils.plotting import Plotter def load_yaml_conf(yamlfile): with open(yamlfile) as f: @@ -16,9 +24,149 @@ def load_yaml_conf(yamlfile): CheckConfiguration(configuration=configuration) return configuration -def configure_simulator(simulator, configuration): +def execute_mapper_no_visibility(configuration, log): + phlist = seeds_to_string_formatter_files(configuration['simulator']['samples'], configuration['simulator']['output'], configuration['simulator']['name'], configuration['simulator']['seed'], 'fits') + skymap = phlist.replace('.fits', '_map.fits').replace(configuration['simulator']['output'], configuration['mapper']['output']) + maproi = get_instrument_fov(configuration['simulator']['array']) + mapper = Mapper(log) + mapper.get_countmap_in_fits(dl3_file=phlist, fitsname=skymap, template=map_template(), maproi=maproi, pixelsize=configuration['mapper']['pixelsize'], trange=[0, configuration['mapper']['exposure']], sigma=configuration['mapper']['smooth']) + del mapper + return skymap + +def configure_simulator_no_visibility(simulator, configuration, log): + if '$TEMPLATES$' in configuration['model']: + configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) simulator.model = configuration['model'] - simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed']) + simulator.output = seeds_to_string_formatter_files(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], 'fits') simulator.caldb = configuration['prod'] - simulator.irf = configuration['irf'] + if configuration['irf'] == 'random': + simulator.irf = select_random_irf(configuration['array'], configuration['prod']) + log.info(f"Randomising instrument response function [{simulator.irf}]") + else: + simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) + simulator.t = [0, configuration['duration']] + simulator.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['array']), simulator.irf) + log.info(f"Verified energy range {simulator.e}") + simulator.seed = configuration['seed'] + simulator.set_log = False + return simulator + +def set_pointing(simulator, configuration, log): + if configuration['pointing'] == 'random': + point = randomise_pointing_sim(configuration) + log.info(f"Randomising pointing coordinates [{point['point_ra']}, {point['point_dec']}]") + else: + point = get_point_source_info(configuration) + simulator.pointing = [point['point_ra'], point['point_dec']] + return simulator, point + +def randomise_pointing_sim(simulator): + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) + source = model_xml.getRaDec() + del model_xml + ra, dec = source[0][0] * u.deg, source[1][0] * u.deg + else: + ra, dec = np.random.uniform(0, 360) * u.deg, np.random.uniform(-90, 90) * u.deg + # use astropy separation + source = SkyCoord(ra, dec, frame='icrs') + position_angle = 45 * u.deg + separation = np.random.random() * get_instrument_fov(simulator['array']) * u.deg + pointing = source.directional_offset_by(position_angle, separation) + return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + +def get_point_source_info(simulator): + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) + source = model_xml.getRaDec() + del model_xml + # use astropy separation + source = SkyCoord(source[0][0] * u.deg, source[1][0] * u.deg, frame='icrs') + pointing = SkyCoord(simulator['pointing']['ra'] * u.deg, simulator['pointing']['dec'] * u.deg, frame='icrs') + separation = source.separation(pointing) + return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + else: + return {'point_ra': simulator['pointing']['ra'], 'point_dec': simulator['pointing']['dec'], 'offset': np.nan, 'source_ra': np.nan, 'source_dec': np.nan} + +def write_simulation_info(simulator, configuration, pointing, datfile, clock): + name = seeds_to_string_formatter(configuration['samples'], configuration['name'], configuration['seed']) + seed = simulator.seed + fov = simulator.fov + tstart, tstop = simulator.t + duration = configuration['duration'] + irf = configuration['irf'] + point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] + if not isfile(datfile): + with open(datfile, 'w+') as f: + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf fov sim_time\n') + with open(datfile, 'a') as f: + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {irf} {fov} {clock}\n') + +def merge_simulation_info(configuration, log): + folder = configuration['output'] + datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f and 'simulator' in f] + merger = join(folder, 'merged_sim_data.dat') + # check merger file + if isfile(merger): + log.warning(f"Merger output already exists, overwrite {merger}") + f = open(merger, 'w+') + f.close() + # collect data + for i, datfile in enumerate(datfiles): + log.info(f"Collect data from {datfile}") + data = pd.read_csv(join(datfile), sep=' ') + if i == 0: + table = data + else: + table = pd.concat([table, data], ignore_index=True) + log.info(f"Lines in data: {len(table)}") + # write merger file + table.to_csv(merger, index=False, header=True, sep=' ', na_rep=np.nan) + +def write_mapping_info(configuration, datfile, clock): + name = seeds_to_string_formatter(configuration['simulator']['samples'], configuration['simulator']['name'], configuration['simulator']['seed']) + seed = configuration['simulator']['seed'] + exposure = configuration['mapper']['exposure'] + center_type = configuration['mapper']['center'] + pixelsize = configuration['mapper']['pixelsize'] + smooth = configuration['mapper']['smooth'] + if not isfile(datfile): + with open(datfile, 'w+') as f: + f.write('name seed exposure center_on pixelsize smooth map_time\n') + with open(datfile, 'a') as f: + f.write(f'{name} {seed} {exposure} {center_type} {pixelsize} {smooth} {clock}\n') + +def merge_data_info(configuration, mode, log): + folder = configuration['output'] + datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f and mode in f] + merger = join(folder, f'merged_{mode}_data.dat') + log.info(f"Merger file: {merger}") + # check merger file + if isfile(merger): + log.warning(f"Merger output already exists, overwrite {merger}") + f = open(merger, 'w+') + f.close() + # collect data + for i, datfile in enumerate(datfiles): + log.info(f"Collect data from {datfile}") + data = pd.read_csv(join(datfile), sep=' ') + if i == 0: + table = data + else: + table = pd.concat([table, data], ignore_index=True) + log.info(f"Lines in data: {len(table)}") + # write merger file + table.to_csv(merger, index=False, header=True, sep=' ', na_rep=np.nan) + +def plot_map(fitsmap, log): + plotmap = fitsmap.replace('.fits', '.png') + plot = Plotter(log) + plot.plot_fits_skymap(fitsmap, plotmap) + del plot + return plotmap + \ No newline at end of file diff --git a/environment.yml b/environment.yml index 9bc21a3995c721a56d47774a1ac236f9365887c2..b94cc6189002fed224807c4361de603aca68cfe7 100644 --- a/environment.yml +++ b/environment.yml @@ -1,170 +1,12 @@ name: astrort -channels: - - anaconda - - defaults - - conda-forge - - cta-observatory dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_kmp_llvm - - astropy=5.1=py38h7deecbd_0 - - asttokens=2.0.5=pyhd3eb1b0_0 - - attrs=22.1.0=py38h06a4308_0 - - backcall=0.2.0=pyhd3eb1b0_0 - - blas=1.0=mkl - - brotli=1.0.9=h5eee18b_7 - - brotli-bin=1.0.9=h5eee18b_7 - - c-ares=1.19.1=h5eee18b_0 - - ca-certificates=2023.01.10=h06a4308_0 - - cfitsio=3.430=hd130d23_1 - - comm=0.1.2=py38h06a4308_0 - - contourpy=1.0.5=py38hdb19cb5_0 - - ctools=2.0.0=py38_1 - - curl=8.2.1=hdbd6064_0 - - cycler=0.11.0=pyhd3eb1b0_0 - - cyrus-sasl=2.1.28=h52b45da_1 - - dbus=1.13.18=hb2f20db_0 - - debugpy=1.6.7=py38h6a678d5_0 - - decorator=5.1.1=pyhd3eb1b0_0 - - exceptiongroup=1.0.4=py38h06a4308_0 - - executing=0.8.3=pyhd3eb1b0_0 - - expat=2.5.0=h6a678d5_0 - - fontconfig=2.14.1=h4c34cd2_2 - - fonttools=4.25.0=pyhd3eb1b0_0 - - freetype=2.12.1=h4a9f257_0 - - gammalib=2.0.0=py38_1 - - giflib=5.2.1=h5eee18b_3 - - glib=2.69.1=he621ea3_2 - - gst-plugins-base=1.14.1=h6a678d5_1 - - gstreamer=1.14.1=h5eee18b_1 - - healpy=1.13.0=py38hf476568_1 - - icu=58.2=he6710b0_3 - - importlib-metadata=6.0.0=py38h06a4308_0 - - importlib_metadata=6.0.0=hd3eb1b0_0 - - importlib_resources=5.2.0=pyhd3eb1b0_1 - - iniconfig=1.1.1=pyhd3eb1b0_0 - - intel-openmp=2023.1.0=hdb19cb5_46305 - - ipykernel=6.25.0=py38h2f386ee_0 - - ipython=8.12.2=py38h06a4308_0 - - jedi=0.18.1=py38h06a4308_1 - - jpeg=9e=h5eee18b_1 - - jupyter_client=8.1.0=py38h06a4308_0 - - jupyter_core=5.3.0=py38h06a4308_0 - - kiwisolver=1.4.4=py38h6a678d5_0 - - krb5=1.20.1=h143b758_1 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libblas=3.9.0=1_h6e990d7_netlib - - libbrotlicommon=1.0.9=h5eee18b_7 - - libbrotlidec=1.0.9=h5eee18b_7 - - libbrotlienc=1.0.9=h5eee18b_7 - - libcblas=3.9.0=3_h893e4fe_netlib - - libclang=14.0.6=default_hc6dbbc7_1 - - libclang13=14.0.6=default_he11475f_1 - - libcups=2.4.2=h2d74bed_1 - - libcurl=8.2.1=h251f7ec_0 - - libdeflate=1.17=h5eee18b_0 - - libedit=3.1.20221030=h5eee18b_0 - - libev=4.33=h7f8727e_1 - - libevent=2.1.12=hdbd6064_1 - - libffi=3.4.4=h6a678d5_0 - - libgcc-ng=13.1.0=he5830b7_0 - - libgfortran=3.0.0=1 - - libgfortran-ng=7.5.0=ha8ba4b0_17 - - libgfortran4=7.5.0=ha8ba4b0_17 - - liblapack=3.9.0=3_h893e4fe_netlib - - libllvm14=14.0.6=hdb19cb5_3 - - libnghttp2=1.52.0=h2d74bed_1 - - libpng=1.6.39=h5eee18b_0 - - libpq=12.15=hdbd6064_1 - - libsodium=1.0.18=h7b6447c_0 - - libssh2=1.10.0=hdbd6064_2 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtiff=4.5.1=h6a678d5_0 - - libuuid=1.41.5=h5eee18b_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - libxcb=1.15=h7f8727e_0 - - libxkbcommon=1.0.1=h5eee18b_1 - - libxml2=2.10.4=hcbfbd50_0 - - libxslt=1.1.37=h2085143_0 - - llvm-openmp=14.0.6=h9e868ea_0 - - lxml=4.9.2=py38h5eee18b_0 - - lz4-c=1.9.4=h6a678d5_0 - - matplotlib=3.7.2=py38h06a4308_0 - - matplotlib-base=3.7.2=py38h1128e8f_0 - - matplotlib-inline=0.1.6=py38h06a4308_0 - - mkl=2023.1.0=h213fc3f_46343 - - mkl-service=2.4.0=py38h5eee18b_1 - - mkl_fft=1.3.6=py38h417a72b_1 - - mkl_random=1.2.2=py38h417a72b_1 - - munkres=1.1.4=py_0 - - mysql=5.7.24=h721c034_2 - - ncurses=6.4=h6a678d5_0 - - nest-asyncio=1.5.6=py38h06a4308_0 - - nspr=4.35=h6a678d5_0 - - nss=3.89.1=h6a678d5_0 - - numpy=1.24.3=py38hf6e8229_1 - - numpy-base=1.24.3=py38h060ed82_1 - - openssl=3.0.10=h7f8727e_2 - - packaging=23.1=py38h06a4308_0 - - pandas=1.3.5=py38h43a58ef_0 - - parso=0.8.3=pyhd3eb1b0_0 - - pcre=8.45=h295c915_0 - - pexpect=4.8.0=pyhd3eb1b0_3 - - pickleshare=0.7.5=pyhd3eb1b0_1003 - - pillow=9.4.0=py38h6a678d5_0 - - pip=23.2.1=py38h06a4308_0 - - platformdirs=3.10.0=py38h06a4308_0 - - pluggy=1.0.0=py38h06a4308_1 - - ply=3.11=py38_0 - - prompt-toolkit=3.0.36=py38h06a4308_0 - - psutil=5.9.0=py38h5eee18b_0 - - ptyprocess=0.7.0=pyhd3eb1b0_2 - - pure_eval=0.2.2=pyhd3eb1b0_0 - - py=1.11.0=pyhd3eb1b0_0 - - pyerfa=2.0.0=py38h27cfd23_0 - - pygments=2.15.1=py38h06a4308_1 - - pyparsing=3.0.9=py38h06a4308_0 - - pyqt=5.15.7=py38h6a678d5_1 - - pyqt5-sip=12.11.0=py38h6a678d5_1 - - pytest=7.1.2=py38h06a4308_0 - - pytest-runner=6.0.0=py38h06a4308_0 - - python=3.8.17=h955ad1f_0 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python_abi=3.8=2_cp38 - - pytz=2022.7=py38h06a4308_0 - - pyyaml=6.0=py38h5eee18b_1 - - pyzmq=25.1.0=py38h6a678d5_0 - - qt-main=5.15.2=h7358343_9 - - qt-webengine=5.15.9=h9ab4d14_7 - - qtwebkit=5.212=h3fafdc1_5 - - readline=8.2=h5eee18b_0 - - regions=0.7=py38h26c90d9_2 - - scipy=1.5.3=py38h828c644_0 - - seaborn=0.12.2=py38h06a4308_0 - - setuptools=59.8.0=py38h578d9bd_1 - - sip=6.6.2=py38h6a678d5_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.2=h5eee18b_0 - - stack_data=0.2.0=pyhd3eb1b0_0 - - tbb=2021.8.0=hdb19cb5_0 - - tk=8.6.12=h1ccaba5_0 - - toml=0.10.2=pyhd3eb1b0_0 - - tomli=2.0.1=py38h06a4308_0 - - tornado=6.3.2=py38h5eee18b_0 - - traitlets=5.7.1=py38h06a4308_0 - - typing_extensions=4.7.1=py38h06a4308_0 - - wcwidth=0.2.5=pyhd3eb1b0_0 - - wheel=0.38.4=py38h06a4308_0 - - xz=5.4.2=h5eee18b_0 - - yaml=0.2.5=h7b6447c_0 - - zeromq=4.3.4=h2531618_0 - - zipp=3.11.0=py38h06a4308_0 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.5=hc292b87_0 + - python=3.8 + - ctools + - astropy + - scipy + - matplotlib + - pandas + - pyyaml + - pytest - pip: - - astrort==0.0.0 - - rtavis==0.0.0 -prefix: /data01/homes/dipiano/.conda/envs/astrort + - lxml diff --git a/rtasci b/rtasci index 20ce02eb59e3598b72a14d85424a6e57ea3a3a55..2dee2a9c5c364558196047ae857508bfcf6e40fe 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 20ce02eb59e3598b72a14d85424a6e57ea3a3a55 +Subproject commit 2dee2a9c5c364558196047ae857508bfcf6e40fe diff --git a/setup.py b/setup.py index d40ff4c9e529ab3a4be69881853bbb04b232b6e4..7bc2f8324e1fa6504135c19a7c2d083da4e81c1d 100644 --- a/setup.py +++ b/setup.py @@ -8,11 +8,19 @@ from setuptools import setup, find_packages +entry_points = { + 'console_scripts': [ + 'base_simulator = astrort.simulator.base_simulator:main', + ] +} + setup( name='astrort', author='Ambra Di Piano ', package_dir={'astrort': 'astrort'}, + entry_points=entry_points, packages=find_packages(), include_package_data=True, license='BSD-3-Clause', + python_requires=">=3.8", ) \ No newline at end of file