From ed50422fa2bc724ed1b6ac0abec3971212eddd4c Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 15:34:21 +0100 Subject: [PATCH 01/22] add mpthreads to argparse --- astrort/simulator/base_simulator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index c33e350..dedc7f9 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -66,6 +66,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='') parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + parser.add_argument('-mp', '--mpthreads', type=int, default=0, choices=range(0,7), help='Number of threads to use for parallel simulation, if unset it will not use multi-threading' ) args = parser.parse_args() main(args.configuration, args.nodes) -- GitLab From 15438db1788952706a57e1e34535e26078a6938a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:21:23 +0100 Subject: [PATCH 02/22] randomise pointing and test --- astrort/simulator/base_simulator.py | 5 ++++- astrort/testing/test_utils/test_wrap.py | 15 ++++++++++++- astrort/utils/wrap.py | 29 +++++++++++++++++-------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index dedc7f9..85644e6 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,7 +10,7 @@ import argparse import pandas as pd from time import time from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf, randomise_target from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission @@ -34,6 +34,9 @@ def base_simulator(configuration_file): # loop seeds for i in range(configuration['simulator']['samples']): clock_sim = time() + # randomise source position in model + if configuration['simulator']['target'] == 'random' and 'background' not in configuration['simulator']['model']: + configuration['simulator']['model'] = randomise_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed']) simulator = RTACtoolsSimulation() # check pointing option if replica is not None: diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 6170ad0..b80e91b 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -189,4 +189,17 @@ def test_set_pointing(test_conf_file, test_data_folder, replicate): log = set_logger(logging.CRITICAL) sim = RTACtoolsSimulation() sim, point = set_pointing(sim, conf, log) - assert type(sim.ra) == type(sim.dec) \ No newline at end of file + assert type(sim.ra) == type(sim.dec) + +@pytest.mark.test_conf_file +@pytest.mark.test_tmp_folder +def test_randomise_target(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + model = conf['simulator']['model'] + new_model = randomise_target(model=model, output=test_tmp_folder, samples=1, name='crab', seed=1) + model_xml = ManageXml(xml=new_model) + source = model_xml.getRaDec() + del model_xml + ra, dec = source[0][0], source[1][0] + assert ra != 83.63 + assert dec != 22.01 diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index d8fe250..0b21cf6 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd import astropy.units as u from os.path import dirname, abspath, join, basename, isfile +from shutil import copyfile from rtasci.lib.RTAManageXml import ManageXml from astropy.coordinates import SkyCoord from astrort.utils.utils import * @@ -61,10 +62,10 @@ def set_pointing(simulator, configuration, log): return simulator, point def randomise_pointing_sim(simulator): - if '$TEMPLATES$' in simulator['model']: - simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) - if 'background.xml' not in simulator['model']: - model_xml = ManageXml(xml=simulator['model']) + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + if 'background.xml' not in model: + model_xml = ManageXml(xml=model) source = model_xml.getRaDec() del model_xml ra, dec = source[0][0] * u.deg, source[1][0] * u.deg @@ -78,10 +79,10 @@ def randomise_pointing_sim(simulator): return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} def get_point_source_info(simulator): - if '$TEMPLATES$' in simulator['model']: - simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) - if 'background.xml' not in simulator['model']: - model_xml = ManageXml(xml=simulator['model']) + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + if 'background.xml' not in model: + model_xml = ManageXml(xml=model) source = model_xml.getRaDec() del model_xml # use astropy separation @@ -175,4 +176,14 @@ def set_irf(configuration, log): log.info(f"Randomising instrument response function [{irf}]") else: irf = configuration['irf'] - return irf \ No newline at end of file + return irf + +def randomise_target(model, output, samples, name, seed): + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + new_model = join(output, seeds_to_string_formatter(samples, name, seed) + '.xml') + copyfile(model, new_model) + model_xml = ManageXml(xml=new_model) + model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(np.random.uniform(0, 360), np.random.uniform(-90, 90)), source=name) + del model_xml + return new_model \ No newline at end of file -- GitLab From 16c72cee5e44d9bbed9f4660310311d3e2f1884c Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:39:05 +0100 Subject: [PATCH 03/22] test randomise source --- astrort/configure/test.yml | 1 + astrort/testing/data/crab_001.xml | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 astrort/testing/data/crab_001.xml diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index be7ca2a..8cf24d6 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -4,6 +4,7 @@ simulator: irf: North_z60_0.5h_LST prod: prod5-v0.1 pointing: random + target: random maxoffset: 2 duration: 10 samples: 2 diff --git a/astrort/testing/data/crab_001.xml b/astrort/testing/data/crab_001.xml new file mode 100644 index 0000000..d4f2696 --- /dev/null +++ b/astrort/testing/data/crab_001.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file -- GitLab From 8d5c7a143309f026a5a46329a1881837dc2154ff Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:39:13 +0100 Subject: [PATCH 04/22] replicate randomise source --- astrort/simulator/base_simulator.py | 6 ++++-- astrort/testing/test_utils/test_wrap.py | 13 +++++++++++++ astrort/utils/wrap.py | 10 ++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 85644e6..bcc0eab 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,7 +10,7 @@ import argparse import pandas as pd from time import time from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf, randomise_target +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf, randomise_target, replicate_target from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission @@ -35,7 +35,7 @@ def base_simulator(configuration_file): for i in range(configuration['simulator']['samples']): clock_sim = time() # randomise source position in model - if configuration['simulator']['target'] == 'random' and 'background' not in configuration['simulator']['model']: + if configuration['simulator']['target'] == 'random' and replica is None: configuration['simulator']['model'] = randomise_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed']) simulator = RTACtoolsSimulation() # check pointing option @@ -43,6 +43,8 @@ def base_simulator(configuration_file): configuration['simulator']['pointing'] = {'ra': replica[replica['seed']==configuration['simulator']['seed']]['point_ra'].values[0], 'dec': replica[replica['seed']==configuration['simulator']['seed']]['point_dec'].values[0]} configuration['simulator']['irf'] = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] + configuration['simulator']['model'] = replicate_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed'], ra=replica[replica['seed']==configuration['simulator']['seed']]['source_ra'].values[0], dec=replica[replica['seed']==configuration['simulator']['seed']]['source_dec'].values[0]) + simulator, point = set_pointing(simulator, configuration['simulator'], log) simulator.irf = set_irf(configuration['simulator'], log) # complete configuration diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index b80e91b..ac8bad3 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -203,3 +203,16 @@ def test_randomise_target(test_conf_file, test_tmp_folder): ra, dec = source[0][0], source[1][0] assert ra != 83.63 assert dec != 22.01 + +@pytest.mark.test_conf_file +@pytest.mark.test_tmp_folder +def test_replicate_target(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + ra, dec = 145.36, -21.92 + new_model = replicate_target(model=conf['simulator']['model'], output=test_tmp_folder, samples=1, name='crab', seed=1, ra=ra, dec=dec) + model_xml = ManageXml(xml=new_model) + source = model_xml.getRaDec() + del model_xml + new_ra, new_dec = source[0][0], source[1][0] + assert np.round(new_ra, decimals=2) == ra + assert np.round(new_dec, decimals=2) == dec diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 0b21cf6..e6c6765 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -186,4 +186,14 @@ def randomise_target(model, output, samples, name, seed): model_xml = ManageXml(xml=new_model) model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(np.random.uniform(0, 360), np.random.uniform(-90, 90)), source=name) del model_xml + return new_model + +def replicate_target(model, output, samples, name, seed, ra, dec): + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + new_model = join(output, seeds_to_string_formatter(samples, name, seed) + '.xml') + copyfile(model, new_model) + model_xml = ManageXml(xml=new_model) + model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(ra, dec), source=name) + del model_xml return new_model \ No newline at end of file -- GitLab From 542ffcc75b3128a34728eee952d117177b171755 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:39:46 +0100 Subject: [PATCH 05/22] update test conf --- astrort/configure/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 8cf24d6..f43aab2 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -4,7 +4,7 @@ simulator: irf: North_z60_0.5h_LST prod: prod5-v0.1 pointing: random - target: random + target: null maxoffset: 2 duration: 10 samples: 2 -- GitLab From 7252449444025826207a259b476863112bc63c7e Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:57:50 +0100 Subject: [PATCH 06/22] fix tests and bugs --- astrort/configure/check_configuration.py | 3 ++- astrort/utils/wrap.py | 16 ++++++++-------- environment.yml | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index 25ebfe6..2ae0a13 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -25,13 +25,14 @@ class CheckConfiguration(): return self def check_simulator(self): - keys = ['name', 'array', 'irf', 'prod', 'pointing', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output'] + keys = ['name', 'array', 'irf', 'prod', 'pointing', 'target', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output'] assert self.conf['simulator'].keys() == keys assert type(self.conf['simulator']['name']) == str assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south'] assert type(self.conf['simulator']['irf']) == str assert type(self.conf['simulator']['prod']) == str assert type(self.conf['simulator']['pointing']) == (str or dict) + assert type(self.conf['simulator']['target']) == (str or None) assert type(self.conf['simulator']['maxoffset']) == (int or float) assert type(self.conf['simulator']['duration']) == int assert type(self.conf['simulator']['samples']) == int diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index e6c6765..6f5cedc 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -62,10 +62,10 @@ def set_pointing(simulator, configuration, log): return simulator, point def randomise_pointing_sim(simulator): - if '$TEMPLATES$' in model: - model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) - if 'background.xml' not in model: - model_xml = ManageXml(xml=model) + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) source = model_xml.getRaDec() del model_xml ra, dec = source[0][0] * u.deg, source[1][0] * u.deg @@ -79,10 +79,10 @@ def randomise_pointing_sim(simulator): return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} def get_point_source_info(simulator): - if '$TEMPLATES$' in model: - model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) - if 'background.xml' not in model: - model_xml = ManageXml(xml=model) + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) source = model_xml.getRaDec() del model_xml # use astropy separation diff --git a/environment.yml b/environment.yml index b94cc61..15d749f 100644 --- a/environment.yml +++ b/environment.yml @@ -8,5 +8,6 @@ dependencies: - pandas - pyyaml - pytest + - pytest-cov - pip: - lxml -- GitLab From e07fb73387e90aadc170caa05565914209aafd6f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:57:55 +0100 Subject: [PATCH 07/22] update --- astrort/configure/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index f43aab2..9d4ff66 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -22,6 +22,7 @@ mapper: region: false output: /data01/homes/dipiano/astroRT/astrort/testing/tmp save: npy + replicate: null visibility: start_time: '2030-01-01T00:00:00' -- GitLab From eb943c552ff28ff9668a5e7815f387b04ce86348 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:58:01 +0100 Subject: [PATCH 08/22] remove unused --- astrort/testing/data/crab_001.xml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 astrort/testing/data/crab_001.xml diff --git a/astrort/testing/data/crab_001.xml b/astrort/testing/data/crab_001.xml deleted file mode 100644 index d4f2696..0000000 --- a/astrort/testing/data/crab_001.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file -- GitLab From 95bd1936084765b0d1f7b4f8f426d55ce24da58c Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 16:58:09 +0100 Subject: [PATCH 09/22] fix bugs --- astrort/testing/test_utils/test_wrap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index ac8bad3..b35ae89 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -176,8 +176,8 @@ def test_set_irf(test_conf_file, test_data_folder, replicate): conf = load_yaml_conf(test_conf_file) conf['simulator']['replicate'] = join(test_data_folder, replicate) if replicate is not None else replicate log = set_logger(logging.CRITICAL) - irf = set_irf(conf, log) - assert conf['array'].upper() in irf + irf = set_irf(conf['simulator'], log) + assert conf['simulator']['array'].upper() in irf @pytest.mark.test_conf_file @pytest.mark.test_data_folder @@ -188,8 +188,8 @@ def test_set_pointing(test_conf_file, test_data_folder, replicate): conf['simulator']['replicate'] = join(test_data_folder, replicate) if replicate is not None else replicate log = set_logger(logging.CRITICAL) sim = RTACtoolsSimulation() - sim, point = set_pointing(sim, conf, log) - assert type(sim.ra) == type(sim.dec) + sim, point = set_pointing(sim, conf['simulator'], log) + assert type(sim.pointing[0]) == type(sim.pointing[1]) @pytest.mark.test_conf_file @pytest.mark.test_tmp_folder -- GitLab From 7fd20bbc949aef968f56daf871b1b57a2de2a220 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 17:16:36 +0100 Subject: [PATCH 10/22] update check conf --- astrort/configure/check_configuration.py | 7 +++++-- astrort/configure/test.yml | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index 2ae0a13..b992169 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -25,7 +25,7 @@ class CheckConfiguration(): return self def check_simulator(self): - keys = ['name', 'array', 'irf', 'prod', 'pointing', 'target', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output'] + keys = ['name', 'array', 'irf', 'prod', 'pointing', 'target', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output', 'replicate'] assert self.conf['simulator'].keys() == keys assert type(self.conf['simulator']['name']) == str assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south'] @@ -39,6 +39,7 @@ class CheckConfiguration(): assert type(self.conf['simulator']['seed']) == int assert type(self.conf['simulator']['model']) == str assert type(self.conf['simulator']['output']) == str + assert type(self.conf['simulator']['replicate']) == (str or None) return self def check_visibility(self): @@ -69,7 +70,7 @@ class CheckConfiguration(): return self def check_mapper(self): - keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output'] + keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output', 'replicate', 'save'] assert self.conf['mapper'].keys() == keys assert type(self.conf['mapper']['exposure']) == int assert type(self.conf['mapper']['smooth']) == (float or int) @@ -78,4 +79,6 @@ class CheckConfiguration(): assert type(self.conf['mapper']['plot']) == bool assert type(self.conf['mapper']['region']) == bool assert type(self.conf['mapper']['output']) == str + assert type(self.conf['simulator']['replicate']) == (str or None) + assert type(self.conf['simulator']['save']) == str return self \ No newline at end of file diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 9d4ff66..2d0d41d 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -21,8 +21,9 @@ mapper: plot: true region: false output: /data01/homes/dipiano/astroRT/astrort/testing/tmp - save: npy replicate: null + save: npy + visibility: start_time: '2030-01-01T00:00:00' -- GitLab From f6e8117e03dda039392131e97284ec037bf060fb Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 17:53:03 +0100 Subject: [PATCH 11/22] update --- rtasci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtasci b/rtasci index 2dee2a9..7a78c66 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 2dee2a9c5c364558196047ae857508bfcf6e40fe +Subproject commit 7a78c662551dba475dacec36efdd6c532eb74f7f -- GitLab From e30c5d52477d3f63886e80b100924df8403bbb5a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 17:53:41 +0100 Subject: [PATCH 12/22] remuve unused --- astrort/simulator/base_simulator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index bcc0eab..bd3a341 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -23,7 +23,6 @@ def base_simulator(configuration_file): log.info(f"Simulator configured, took {time() - clock} s") # create output dir log.info(f"Output folder: {configuration['simulator']['output']}") - #makedirs(configuration['simulator']['output'], exist_ok=True) # start simulations log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") if configuration['simulator']['replicate'] is not None: -- GitLab From f92d34b059fb2faa4043f3e7981edbb378eff447 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Dec 2023 17:53:51 +0100 Subject: [PATCH 13/22] create maps for cleaner cnn --- astrort/simulator/image_cleaning_mapper.py | 93 ++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 astrort/simulator/image_cleaning_mapper.py diff --git a/astrort/simulator/image_cleaning_mapper.py b/astrort/simulator/image_cleaning_mapper.py new file mode 100644 index 0000000..6468cd9 --- /dev/null +++ b/astrort/simulator/image_cleaning_mapper.py @@ -0,0 +1,93 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +import pandas as pd +from time import time +from os import makedirs +from os.path import join +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility, plot_map +from astrort.utils.utils import get_all_seeds, get_instrument_fov, get_instrument_tev_range, adjust_tev_range_to_irf +from astrort.configure.logging import set_logger, get_log_level, get_logfile +from astrort.configure.slurmjobs import slurm_submission +from rtasci.lib.RTACtoolsAnalysis import RTACtoolsAnalysis + +def base_mapper(configuration_file, seeds=None): + clock = time() + configuration = load_yaml_conf(configuration_file) + logfile = get_logfile(configuration, mode='mapper') + datfile = logfile.replace('.log', '.dat') + # set logger + log = set_logger(get_log_level(configuration['logging']['level']), logfile) + # collect simulations to map + if seeds is None: + log.info(f"Mapping of all simulations found") + seeds = get_all_seeds(configuration['simulator']) + log.info(f"Mapper configured, took {time() - clock} s") + # create output dir + log.info(f"Output folder: {configuration['mapper']['output']}") + makedirs(configuration['mapper']['output'], exist_ok=True) + # start mapping + log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") + if configuration['mapper']['replicate'] is not None: + replica = pd.read_csv(configuration['mapper']['replicate'], sep=' ', header=0) + log.info(f"Replicate pointing and IRF from {configuration['mapper']['replicate']}") + else: + raise ValueError(f'This script requires a "mapper:replicate" configuration other than None') + for seed in seeds: + clock_map = time() + # configure map + mapper = RTACtoolsAnalysis() + mapper.input = join(configuration['simulator']['output'], replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '.fits') + mapper.irf = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] + mapper.caldb = configuration['mapper']['caldb'] + mapper.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['array']), mapper.irf) + mapper.t = [0, configuration['mapper']['exposure']] + mapper.roi = get_instrument_fov(configuration['simulator']['array']) + # make noisy map + mapper.sky_subtraction = 'NONE' + mapper.output = join(configuration['mapper']['output'] + '_noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + # make plot + if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': + clock_plot = time() + plotmap = plot_map(mapper.output, log) + log.info(f"Plotting noisy image (seed = {seed}) complete, took {time() - clock_plot} s") + # make clean map + mapper.sky_subtraction = 'IRF' + mapper.output = join(configuration['mapper']['output'] + '_clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + # make plot + if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': + clock_plot = time() + plotmap = plot_map(mapper.output, log) + log.info(f"Plotting clean image (seed = {seed}) complete, took {time() - clock_plot} s") + del mapper + log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s") + # timing simulation + clock_map = time() - clock_map + # save simulation data + write_mapping_info(configuration, datfile, clock_map) + configuration['simulator']['seed'] += 1 + # end simulations + log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n") + log.info(f"Process complete, took {time() - clock} s") + +def main(configuration, nodes): + if nodes == 0: + base_mapper(configuration) + else: + slurm_submission(configuration, nodes, mode='mapper') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + args = parser.parse_args() + + main(args.configuration, args.nodes) \ No newline at end of file -- GitLab From 4eedda9ffe61520dcd8c76f07b97ce052188cda1 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 18 Dec 2023 16:00:36 +0100 Subject: [PATCH 14/22] rename --- .../{image_cleaning_mapper.py => base_cleaner.py} | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) rename astrort/simulator/{image_cleaning_mapper.py => base_cleaner.py} (94%) diff --git a/astrort/simulator/image_cleaning_mapper.py b/astrort/simulator/base_cleaner.py similarity index 94% rename from astrort/simulator/image_cleaning_mapper.py rename to astrort/simulator/base_cleaner.py index 6468cd9..2c693ec 100644 --- a/astrort/simulator/image_cleaning_mapper.py +++ b/astrort/simulator/base_cleaner.py @@ -11,13 +11,13 @@ import pandas as pd from time import time from os import makedirs from os.path import join -from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility, plot_map +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, plot_map from astrort.utils.utils import get_all_seeds, get_instrument_fov, get_instrument_tev_range, adjust_tev_range_to_irf from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission from rtasci.lib.RTACtoolsAnalysis import RTACtoolsAnalysis -def base_mapper(configuration_file, seeds=None): +def base_cleaner(configuration_file, seeds=None): clock = time() configuration = load_yaml_conf(configuration_file) logfile = get_logfile(configuration, mode='mapper') @@ -52,6 +52,7 @@ def base_mapper(configuration_file, seeds=None): # make noisy map mapper.sky_subtraction = 'NONE' mapper.output = join(configuration['mapper']['output'] + '_noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + log.debug(f"Noisy map: {mapper.output}") mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) # make plot if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': @@ -62,6 +63,7 @@ def base_mapper(configuration_file, seeds=None): mapper.sky_subtraction = 'IRF' mapper.output = join(configuration['mapper']['output'] + '_clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + log.debug(f"Clean map: {mapper.output}") # make plot if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': clock_plot = time() @@ -80,9 +82,9 @@ def base_mapper(configuration_file, seeds=None): def main(configuration, nodes): if nodes == 0: - base_mapper(configuration) + base_cleaner(configuration) else: - slurm_submission(configuration, nodes, mode='mapper') + slurm_submission(configuration, nodes, mode='mapper', script='base_cleaner') if __name__ == '__main__': parser = argparse.ArgumentParser(description='') -- GitLab From 17e57e32d7b7b49d1d85ce41c08cabd13f06db8c Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 18 Dec 2023 16:06:49 +0100 Subject: [PATCH 15/22] skip galactic plane --- astrort/utils/wrap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 6f5cedc..77ac245 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -184,7 +184,8 @@ def randomise_target(model, output, samples, name, seed): new_model = join(output, seeds_to_string_formatter(samples, name, seed) + '.xml') copyfile(model, new_model) model_xml = ManageXml(xml=new_model) - model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(np.random.uniform(0, 360), np.random.uniform(-90, 90)), source=name) + # skip DEC galactic center +/- 30 deg (DEC allowed [-90, -60] U [0, +90] in GAL [-90, -30] U [30, 90]) + model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(np.random.uniform(0, 360), np.random.choice([np.random.uniform(-90, 60), np.random.uniform(0, 90)])), source=name) del model_xml return new_model -- GitLab From e5593ab8085b4850668a6231ebc4b83e777d6018 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 18 Dec 2023 16:06:58 +0100 Subject: [PATCH 16/22] add script --- astrort/configure/slurmjobs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index f1b6316..683fe5e 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -24,7 +24,7 @@ def make_configuration(jobname_conf, configuration, node_number, mode): with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode, script=None): # write sbatch with open(jobname_sh, 'w+') as f: f.write("#!/bin/bash") @@ -40,21 +40,23 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): f.write(f"\nsource activate {slurmconf['environment']}") if mode == 'simulator': f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") - elif mode == 'mapper': + elif mode == 'mapper' and script is None: f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n") + elif mode == 'mapper' and script is not None: + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), f'{script}.py')} -f {jobname_conf}\n") else: raise ValueError(f"Invalid 'mode' {mode}") -def make_sbatch(jobname, configuration, node_number, mode): +def make_sbatch(jobname, configuration, node_number, mode, script=None): output = configuration[mode]['output'] jobname_sh = join(output, f"{jobname}_{mode}.sh") jobname_log = join(output, f"{jobname}_{mode}.slurm") jobname_conf = join(output, f"{jobname}_{mode}.yml") make_configuration(jobname_conf, configuration, node_number, mode=mode) - make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode) + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode, script=script) system(f"sbatch {jobname_sh}") -def slurm_submission(configuration_file, nodes, mode): +def slurm_submission(configuration_file, nodes, mode, script=None): configuration = load_yaml_conf(configuration_file) log = set_logger(get_log_level(configuration['logging']['level'])) # create output dir @@ -64,4 +66,4 @@ def slurm_submission(configuration_file, nodes, mode): configuration['slurm']['nodes'] = nodes for node_number in range(configuration['slurm']['nodes']): jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_sbatch(jobname, configuration, node_number, mode=mode) + make_sbatch(jobname, configuration, node_number, mode=mode, script=script) -- GitLab From 199ce7d142872ed5bf938c58b4c2ba2239cc6d31 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 18 Dec 2023 16:09:21 +0100 Subject: [PATCH 17/22] formatting up to threshold --- astrort/utils/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 4b3a8d9..3ba0078 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -15,11 +15,11 @@ def map_template(): return join(dirname(abspath(__file__)).replace('utils', 'templates'), 'base_empty_map.fits') def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=None): - if samples <= 1e3: + if samples < 1e3: name = join(output, f"{name}_{seed:03d}.{ext}") - elif samples <= 1e5: + elif samples < 1e5: name = join(output, f"{name}_{seed:05d}.{ext}") - elif samples <= 1e8: + elif samples < 1e8: name = join(output, f"{name}_{seed:08d}.{ext}") else: name = join(output, f"{name}_{seed}.{ext}") @@ -29,11 +29,11 @@ def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=Non return name def seeds_to_string_formatter(samples, name, seed): - if samples <= 1e3: + if samples < 1e3: name = join(f"{name}_{seed:03d}") - elif samples <= 1e5: + elif samples < 1e5: name = join(f"{name}_{seed:05d}") - elif samples <= 1e8: + elif samples < 1e8: name = join(f"{name}_{seed:08d}") else: name = join(f"{name}_{seed}") -- GitLab From c1412f2c1eec28223ef2c6a370395577d2d86f24 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 19 Dec 2023 15:41:35 +0100 Subject: [PATCH 18/22] update --- rtasci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtasci b/rtasci index 7a78c66..0be1a02 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 7a78c662551dba475dacec36efdd6c532eb74f7f +Subproject commit 0be1a028b9f4d52979aacd82ab8e0a140d337c60 -- GitLab From c424e1e81fb33b2af712eae3246965e6b2de9664 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 19 Dec 2023 15:41:45 +0100 Subject: [PATCH 19/22] fix replica --- astrort/configure/slurmjobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 683fe5e..909ce95 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -19,7 +19,7 @@ def make_configuration(jobname_conf, configuration, node_number, mode): configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log') configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat') if configuration[mode]['replicate'] is not None: - configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_{mode}.dat') + configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_simulator.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -- GitLab From 3374f2d75cf9d250725e3dfa537109042c9df911 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 19 Dec 2023 15:42:09 +0100 Subject: [PATCH 20/22] fix folders --- astrort/simulator/base_cleaner.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/astrort/simulator/base_cleaner.py b/astrort/simulator/base_cleaner.py index 2c693ec..6ab5ba3 100644 --- a/astrort/simulator/base_cleaner.py +++ b/astrort/simulator/base_cleaner.py @@ -32,6 +32,8 @@ def base_cleaner(configuration_file, seeds=None): # create output dir log.info(f"Output folder: {configuration['mapper']['output']}") makedirs(configuration['mapper']['output'], exist_ok=True) + makedirs(join(configuration['mapper']['output'], 'clean'), exist_ok=True) + makedirs(join(configuration['mapper']['output'], 'noisy'), exist_ok=True) # start mapping log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") if configuration['mapper']['replicate'] is not None: @@ -44,16 +46,18 @@ def base_cleaner(configuration_file, seeds=None): # configure map mapper = RTACtoolsAnalysis() mapper.input = join(configuration['simulator']['output'], replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '.fits') + log.debug(f"DL3: {mapper.input}") mapper.irf = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] - mapper.caldb = configuration['mapper']['caldb'] - mapper.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['array']), mapper.irf) + log.debug(f"IRF: {mapper.irf}") + mapper.caldb = configuration['simulator']['prod'] + mapper.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['simulator']['array']), mapper.irf) mapper.t = [0, configuration['mapper']['exposure']] mapper.roi = get_instrument_fov(configuration['simulator']['array']) # make noisy map mapper.sky_subtraction = 'NONE' - mapper.output = join(configuration['mapper']['output'] + '_noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') - log.debug(f"Noisy map: {mapper.output}") + mapper.output = join(configuration['mapper']['output'], 'noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + log.debug(f"Noisy map: {mapper.output}") # make plot if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': clock_plot = time() @@ -61,7 +65,7 @@ def base_cleaner(configuration_file, seeds=None): log.info(f"Plotting noisy image (seed = {seed}) complete, took {time() - clock_plot} s") # make clean map mapper.sky_subtraction = 'IRF' - mapper.output = join(configuration['mapper']['output'] + '_clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + mapper.output = join(configuration['mapper']['output'], 'clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) log.debug(f"Clean map: {mapper.output}") # make plot -- GitLab From 99ff17b094dc972ec70a942c46ccce0863a9c661 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 19 Dec 2023 15:42:15 +0100 Subject: [PATCH 21/22] add test --- .../test_simulator/test_base_cleaner.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 astrort/testing/test_simulator/test_base_cleaner.py diff --git a/astrort/testing/test_simulator/test_base_cleaner.py b/astrort/testing/test_simulator/test_base_cleaner.py new file mode 100644 index 0000000..0fe30f8 --- /dev/null +++ b/astrort/testing/test_simulator/test_base_cleaner.py @@ -0,0 +1,46 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import numpy as np +from yaml import dump +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator +from astrort.simulator.base_cleaner import base_cleaner +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +@pytest.mark.test_data_folder +def test_base_cleaner(test_conf_file, test_data_folder): + + # clean output + conf = load_yaml_conf(test_conf_file) + conf['mapper']['save'] = 'fits' + conf['mapper']['replicate'] = join(test_data_folder, 'test_simulator.dat') + rmtree(conf['mapper']['output'], ignore_errors=True) + + # run simulator + base_simulator(test_conf_file) + + # write new mapper configuration + update_conf_file = join(conf['mapper']['output'], 'tmp.yml') + with open(update_conf_file, 'w+') as f: + dump(conf, f, default_flow_style=False) + base_cleaner(update_conf_file) + + # check output + expected_maps = conf['simulator']['samples'] + # noisy map + found_maps = len([f for f in listdir(join(conf['mapper']['output'], 'noisy')) if isfile(join(conf['mapper']['output'], 'noisy', f)) and conf['mapper']['save'] in f and conf['simulator']['name'] in f and 'map' in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + # clean map + found_maps = len([f for f in listdir(join(conf['mapper']['output'], 'clean')) if isfile(join(conf['mapper']['output'], 'clean', f)) and conf['mapper']['save'] in f and conf['simulator']['name'] in f and 'map' in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + -- GitLab From f57572ccfbefa0c9c59df0a5e53bfd2da3cab9fa Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 19 Dec 2023 15:42:21 +0100 Subject: [PATCH 22/22] fix irf --- astrort/utils/wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 77ac245..5496768 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -99,7 +99,7 @@ def write_simulation_info(simulator, configuration, pointing, datfile, clock): fov = simulator.fov tstart, tstop = simulator.t duration = configuration['duration'] - irf = configuration['irf'] + irf = simulator.irf point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: -- GitLab