diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index 25ebfe6bdb0c41528dfe3840ad1fd7a2dc498481..b99216904caba4ec188bb799632056f6ad0b21c3 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -25,19 +25,21 @@ class CheckConfiguration(): return self def check_simulator(self): - keys = ['name', 'array', 'irf', 'prod', 'pointing', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output'] + keys = ['name', 'array', 'irf', 'prod', 'pointing', 'target', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output', 'replicate'] assert self.conf['simulator'].keys() == keys assert type(self.conf['simulator']['name']) == str assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south'] assert type(self.conf['simulator']['irf']) == str assert type(self.conf['simulator']['prod']) == str assert type(self.conf['simulator']['pointing']) == (str or dict) + assert type(self.conf['simulator']['target']) == (str or None) assert type(self.conf['simulator']['maxoffset']) == (int or float) assert type(self.conf['simulator']['duration']) == int assert type(self.conf['simulator']['samples']) == int assert type(self.conf['simulator']['seed']) == int assert type(self.conf['simulator']['model']) == str assert type(self.conf['simulator']['output']) == str + assert type(self.conf['simulator']['replicate']) == (str or None) return self def check_visibility(self): @@ -68,7 +70,7 @@ class CheckConfiguration(): return self def check_mapper(self): - keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output'] + keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output', 'replicate', 'save'] assert self.conf['mapper'].keys() == keys assert type(self.conf['mapper']['exposure']) == int assert type(self.conf['mapper']['smooth']) == (float or int) @@ -77,4 +79,6 @@ class CheckConfiguration(): assert type(self.conf['mapper']['plot']) == bool assert type(self.conf['mapper']['region']) == bool assert type(self.conf['mapper']['output']) == str + assert type(self.conf['simulator']['replicate']) == (str or None) + assert type(self.conf['simulator']['save']) == str return self \ No newline at end of file diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index f1b63165354d17512e6b42d66a31787fb9407685..909ce95868da25aeee95c76ddad191d6236f72db 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -19,12 +19,12 @@ def make_configuration(jobname_conf, configuration, node_number, mode): configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log') configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat') if configuration[mode]['replicate'] is not None: - configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_{mode}.dat') + configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_simulator.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode, script=None): # write sbatch with open(jobname_sh, 'w+') as f: f.write("#!/bin/bash") @@ -40,21 +40,23 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): f.write(f"\nsource activate {slurmconf['environment']}") if mode == 'simulator': f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") - elif mode == 'mapper': + elif mode == 'mapper' and script is None: f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n") + elif mode == 'mapper' and script is not None: + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), f'{script}.py')} -f {jobname_conf}\n") else: raise ValueError(f"Invalid 'mode' {mode}") -def make_sbatch(jobname, configuration, node_number, mode): +def make_sbatch(jobname, configuration, node_number, mode, script=None): output = configuration[mode]['output'] jobname_sh = join(output, f"{jobname}_{mode}.sh") jobname_log = join(output, f"{jobname}_{mode}.slurm") jobname_conf = join(output, f"{jobname}_{mode}.yml") make_configuration(jobname_conf, configuration, node_number, mode=mode) - make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode) + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode, script=script) system(f"sbatch {jobname_sh}") -def slurm_submission(configuration_file, nodes, mode): +def slurm_submission(configuration_file, nodes, mode, script=None): configuration = load_yaml_conf(configuration_file) log = set_logger(get_log_level(configuration['logging']['level'])) # create output dir @@ -64,4 +66,4 @@ def slurm_submission(configuration_file, nodes, mode): configuration['slurm']['nodes'] = nodes for node_number in range(configuration['slurm']['nodes']): jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_sbatch(jobname, configuration, node_number, mode=mode) + make_sbatch(jobname, configuration, node_number, mode=mode, script=script) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index be7ca2a0aa26ba25559af9a67650faa5451cbb78..2d0d41dd3659dd09d1089f3ee640a02c4c2151d3 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -4,6 +4,7 @@ simulator: irf: North_z60_0.5h_LST prod: prod5-v0.1 pointing: random + target: null maxoffset: 2 duration: 10 samples: 2 @@ -20,8 +21,10 @@ mapper: plot: true region: false output: /data01/homes/dipiano/astroRT/astrort/testing/tmp + replicate: null save: npy + visibility: start_time: '2030-01-01T00:00:00' diff --git a/astrort/simulator/base_cleaner.py b/astrort/simulator/base_cleaner.py new file mode 100644 index 0000000000000000000000000000000000000000..6ab5ba38769a8e88ad320db0cb162005baa2b2ef --- /dev/null +++ b/astrort/simulator/base_cleaner.py @@ -0,0 +1,99 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +import pandas as pd +from time import time +from os import makedirs +from os.path import join +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, plot_map +from astrort.utils.utils import get_all_seeds, get_instrument_fov, get_instrument_tev_range, adjust_tev_range_to_irf +from astrort.configure.logging import set_logger, get_log_level, get_logfile +from astrort.configure.slurmjobs import slurm_submission +from rtasci.lib.RTACtoolsAnalysis import RTACtoolsAnalysis + +def base_cleaner(configuration_file, seeds=None): + clock = time() + configuration = load_yaml_conf(configuration_file) + logfile = get_logfile(configuration, mode='mapper') + datfile = logfile.replace('.log', '.dat') + # set logger + log = set_logger(get_log_level(configuration['logging']['level']), logfile) + # collect simulations to map + if seeds is None: + log.info(f"Mapping of all simulations found") + seeds = get_all_seeds(configuration['simulator']) + log.info(f"Mapper configured, took {time() - clock} s") + # create output dir + log.info(f"Output folder: {configuration['mapper']['output']}") + makedirs(configuration['mapper']['output'], exist_ok=True) + makedirs(join(configuration['mapper']['output'], 'clean'), exist_ok=True) + makedirs(join(configuration['mapper']['output'], 'noisy'), exist_ok=True) + # start mapping + log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") + if configuration['mapper']['replicate'] is not None: + replica = pd.read_csv(configuration['mapper']['replicate'], sep=' ', header=0) + log.info(f"Replicate pointing and IRF from {configuration['mapper']['replicate']}") + else: + raise ValueError(f'This script requires a "mapper:replicate" configuration other than None') + for seed in seeds: + clock_map = time() + # configure map + mapper = RTACtoolsAnalysis() + mapper.input = join(configuration['simulator']['output'], replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '.fits') + log.debug(f"DL3: {mapper.input}") + mapper.irf = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] + log.debug(f"IRF: {mapper.irf}") + mapper.caldb = configuration['simulator']['prod'] + mapper.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['simulator']['array']), mapper.irf) + mapper.t = [0, configuration['mapper']['exposure']] + mapper.roi = get_instrument_fov(configuration['simulator']['array']) + # make noisy map + mapper.sky_subtraction = 'NONE' + mapper.output = join(configuration['mapper']['output'], 'noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + log.debug(f"Noisy map: {mapper.output}") + # make plot + if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': + clock_plot = time() + plotmap = plot_map(mapper.output, log) + log.info(f"Plotting noisy image (seed = {seed}) complete, took {time() - clock_plot} s") + # make clean map + mapper.sky_subtraction = 'IRF' + mapper.output = join(configuration['mapper']['output'], 'clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits') + mapper.run_skymap(wbin=configuration['mapper']['pixelsize']) + log.debug(f"Clean map: {mapper.output}") + # make plot + if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits': + clock_plot = time() + plotmap = plot_map(mapper.output, log) + log.info(f"Plotting clean image (seed = {seed}) complete, took {time() - clock_plot} s") + del mapper + log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s") + # timing simulation + clock_map = time() - clock_map + # save simulation data + write_mapping_info(configuration, datfile, clock_map) + configuration['simulator']['seed'] += 1 + # end simulations + log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n") + log.info(f"Process complete, took {time() - clock} s") + +def main(configuration, nodes): + if nodes == 0: + base_cleaner(configuration) + else: + slurm_submission(configuration, nodes, mode='mapper', script='base_cleaner') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + args = parser.parse_args() + + main(args.configuration, args.nodes) \ No newline at end of file diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index c33e350d1f3c342f127db87e394e7b84b8b03b42..bd3a341cfaa821b5debd4b3b028eb61be130a989 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,7 +10,7 @@ import argparse import pandas as pd from time import time from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf, randomise_target, replicate_target from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission @@ -23,7 +23,6 @@ def base_simulator(configuration_file): log.info(f"Simulator configured, took {time() - clock} s") # create output dir log.info(f"Output folder: {configuration['simulator']['output']}") - #makedirs(configuration['simulator']['output'], exist_ok=True) # start simulations log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") if configuration['simulator']['replicate'] is not None: @@ -34,12 +33,17 @@ def base_simulator(configuration_file): # loop seeds for i in range(configuration['simulator']['samples']): clock_sim = time() + # randomise source position in model + if configuration['simulator']['target'] == 'random' and replica is None: + configuration['simulator']['model'] = randomise_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed']) simulator = RTACtoolsSimulation() # check pointing option if replica is not None: configuration['simulator']['pointing'] = {'ra': replica[replica['seed']==configuration['simulator']['seed']]['point_ra'].values[0], 'dec': replica[replica['seed']==configuration['simulator']['seed']]['point_dec'].values[0]} configuration['simulator']['irf'] = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] + configuration['simulator']['model'] = replicate_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed'], ra=replica[replica['seed']==configuration['simulator']['seed']]['source_ra'].values[0], dec=replica[replica['seed']==configuration['simulator']['seed']]['source_dec'].values[0]) + simulator, point = set_pointing(simulator, configuration['simulator'], log) simulator.irf = set_irf(configuration['simulator'], log) # complete configuration @@ -66,6 +70,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='') parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + parser.add_argument('-mp', '--mpthreads', type=int, default=0, choices=range(0,7), help='Number of threads to use for parallel simulation, if unset it will not use multi-threading' ) args = parser.parse_args() main(args.configuration, args.nodes) diff --git a/astrort/testing/test_simulator/test_base_cleaner.py b/astrort/testing/test_simulator/test_base_cleaner.py new file mode 100644 index 0000000000000000000000000000000000000000..0fe30f89867e9cef20177a949dbb64b12640aff9 --- /dev/null +++ b/astrort/testing/test_simulator/test_base_cleaner.py @@ -0,0 +1,46 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import numpy as np +from yaml import dump +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator +from astrort.simulator.base_cleaner import base_cleaner +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +@pytest.mark.test_data_folder +def test_base_cleaner(test_conf_file, test_data_folder): + + # clean output + conf = load_yaml_conf(test_conf_file) + conf['mapper']['save'] = 'fits' + conf['mapper']['replicate'] = join(test_data_folder, 'test_simulator.dat') + rmtree(conf['mapper']['output'], ignore_errors=True) + + # run simulator + base_simulator(test_conf_file) + + # write new mapper configuration + update_conf_file = join(conf['mapper']['output'], 'tmp.yml') + with open(update_conf_file, 'w+') as f: + dump(conf, f, default_flow_style=False) + base_cleaner(update_conf_file) + + # check output + expected_maps = conf['simulator']['samples'] + # noisy map + found_maps = len([f for f in listdir(join(conf['mapper']['output'], 'noisy')) if isfile(join(conf['mapper']['output'], 'noisy', f)) and conf['mapper']['save'] in f and conf['simulator']['name'] in f and 'map' in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + # clean map + found_maps = len([f for f in listdir(join(conf['mapper']['output'], 'clean')) if isfile(join(conf['mapper']['output'], 'clean', f)) and conf['mapper']['save'] in f and conf['simulator']['name'] in f and 'map' in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 6170ad00db96499d182be98021ae29d019be87fe..b35ae89eb5b4edd370b945c662bba058a5a19e16 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -176,8 +176,8 @@ def test_set_irf(test_conf_file, test_data_folder, replicate): conf = load_yaml_conf(test_conf_file) conf['simulator']['replicate'] = join(test_data_folder, replicate) if replicate is not None else replicate log = set_logger(logging.CRITICAL) - irf = set_irf(conf, log) - assert conf['array'].upper() in irf + irf = set_irf(conf['simulator'], log) + assert conf['simulator']['array'].upper() in irf @pytest.mark.test_conf_file @pytest.mark.test_data_folder @@ -188,5 +188,31 @@ def test_set_pointing(test_conf_file, test_data_folder, replicate): conf['simulator']['replicate'] = join(test_data_folder, replicate) if replicate is not None else replicate log = set_logger(logging.CRITICAL) sim = RTACtoolsSimulation() - sim, point = set_pointing(sim, conf, log) - assert type(sim.ra) == type(sim.dec) \ No newline at end of file + sim, point = set_pointing(sim, conf['simulator'], log) + assert type(sim.pointing[0]) == type(sim.pointing[1]) + +@pytest.mark.test_conf_file +@pytest.mark.test_tmp_folder +def test_randomise_target(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + model = conf['simulator']['model'] + new_model = randomise_target(model=model, output=test_tmp_folder, samples=1, name='crab', seed=1) + model_xml = ManageXml(xml=new_model) + source = model_xml.getRaDec() + del model_xml + ra, dec = source[0][0], source[1][0] + assert ra != 83.63 + assert dec != 22.01 + +@pytest.mark.test_conf_file +@pytest.mark.test_tmp_folder +def test_replicate_target(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + ra, dec = 145.36, -21.92 + new_model = replicate_target(model=conf['simulator']['model'], output=test_tmp_folder, samples=1, name='crab', seed=1, ra=ra, dec=dec) + model_xml = ManageXml(xml=new_model) + source = model_xml.getRaDec() + del model_xml + new_ra, new_dec = source[0][0], source[1][0] + assert np.round(new_ra, decimals=2) == ra + assert np.round(new_dec, decimals=2) == dec diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 4b3a8d99eec0830159f77011ae5c64a09af01db3..3ba007829af96ef50bb386681387f9623abbd5d3 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -15,11 +15,11 @@ def map_template(): return join(dirname(abspath(__file__)).replace('utils', 'templates'), 'base_empty_map.fits') def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=None): - if samples <= 1e3: + if samples < 1e3: name = join(output, f"{name}_{seed:03d}.{ext}") - elif samples <= 1e5: + elif samples < 1e5: name = join(output, f"{name}_{seed:05d}.{ext}") - elif samples <= 1e8: + elif samples < 1e8: name = join(output, f"{name}_{seed:08d}.{ext}") else: name = join(output, f"{name}_{seed}.{ext}") @@ -29,11 +29,11 @@ def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=Non return name def seeds_to_string_formatter(samples, name, seed): - if samples <= 1e3: + if samples < 1e3: name = join(f"{name}_{seed:03d}") - elif samples <= 1e5: + elif samples < 1e5: name = join(f"{name}_{seed:05d}") - elif samples <= 1e8: + elif samples < 1e8: name = join(f"{name}_{seed:08d}") else: name = join(f"{name}_{seed}") diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index d8fe2509c8efcdf59c926fa8c89a32ce52940a40..5496768be6d3a39a35c844766cc7f9da06dcf98e 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd import astropy.units as u from os.path import dirname, abspath, join, basename, isfile +from shutil import copyfile from rtasci.lib.RTAManageXml import ManageXml from astropy.coordinates import SkyCoord from astrort.utils.utils import * @@ -98,7 +99,7 @@ def write_simulation_info(simulator, configuration, pointing, datfile, clock): fov = simulator.fov tstart, tstop = simulator.t duration = configuration['duration'] - irf = configuration['irf'] + irf = simulator.irf point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: @@ -175,4 +176,25 @@ def set_irf(configuration, log): log.info(f"Randomising instrument response function [{irf}]") else: irf = configuration['irf'] - return irf \ No newline at end of file + return irf + +def randomise_target(model, output, samples, name, seed): + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + new_model = join(output, seeds_to_string_formatter(samples, name, seed) + '.xml') + copyfile(model, new_model) + model_xml = ManageXml(xml=new_model) + # skip DEC galactic center +/- 30 deg (DEC allowed [-90, -60] U [0, +90] in GAL [-90, -30] U [30, 90]) + model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(np.random.uniform(0, 360), np.random.choice([np.random.uniform(-90, 60), np.random.uniform(0, 90)])), source=name) + del model_xml + return new_model + +def replicate_target(model, output, samples, name, seed, ra, dec): + if '$TEMPLATES$' in model: + model = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(model)) + new_model = join(output, seeds_to_string_formatter(samples, name, seed) + '.xml') + copyfile(model, new_model) + model_xml = ManageXml(xml=new_model) + model_xml.setModelParameters(parameters=('RA', 'DEC'), values=(ra, dec), source=name) + del model_xml + return new_model \ No newline at end of file diff --git a/environment.yml b/environment.yml index b94cc6189002fed224807c4361de603aca68cfe7..15d749f96096ecfeee52068bc8888fa36f1051f6 100644 --- a/environment.yml +++ b/environment.yml @@ -8,5 +8,6 @@ dependencies: - pandas - pyyaml - pytest + - pytest-cov - pip: - lxml diff --git a/rtasci b/rtasci index 2dee2a9c5c364558196047ae857508bfcf6e40fe..0be1a028b9f4d52979aacd82ab8e0a140d337c60 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 2dee2a9c5c364558196047ae857508bfcf6e40fe +Subproject commit 0be1a028b9f4d52979aacd82ab8e0a140d337c60