From 8f0d27d14ae9a7197eb113a1a2a4e8643dee5846 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:37:01 +0200 Subject: [PATCH 01/98] update output --- astrort/configure/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 5728707..5a98e5a 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -8,7 +8,7 @@ simulator: samples: 5 seed: 1 model: $TEMPLATES$/crab.xml - output: /data01/homes/dipiano/astroRT/ + output: /data01/homes/dipiano/astroRT/testing/output_tmp visibility: start_time: '2030-01-01T00:00:00' -- GitLab From ca7d873ceb4f93c1ea22f39f933b6eb7713b54d7 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:47:53 +0200 Subject: [PATCH 02/98] skip for now --- .../testing/test_astrort/test_simulator/test_base_simulator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py b/astrort/testing/test_astrort/test_simulator/test_base_simulator.py index b209240..0d184e9 100644 --- a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py +++ b/astrort/testing/test_astrort/test_simulator/test_base_simulator.py @@ -8,8 +8,9 @@ import pytest from astrort.utils.wrap import load_yaml_conf -from astrort.cfg.check_configuration import CheckConfiguration +from astrort.configure.check_configuration import CheckConfiguration +@pytest.mark.skip('to-do') @pytest.mark.rtadeep_configuration class TestBaseSimulator: -- GitLab From 650097f7eb7babf6086915c30eafd78ed51bc99d Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:48:00 +0200 Subject: [PATCH 03/98] add tmp marker --- astrort/testing/pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astrort/testing/pytest.ini b/astrort/testing/pytest.ini index 5c05197..c37ccaa 100644 --- a/astrort/testing/pytest.ini +++ b/astrort/testing/pytest.ini @@ -1,3 +1,4 @@ [pytest] markers = - astrort_configuration: rtadeep test configurtion file \ No newline at end of file + astrort_configuration: test configurtion file + astrort_tmp_folder: test tmp output folder \ No newline at end of file -- GitLab From c5024ab3dd4894c989f0f640da937a6b0550847f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:48:18 +0200 Subject: [PATCH 04/98] add instrument erange --- astrort/utils/utils.py | 13 ++++++++++++- astrort/utils/wrap.py | 4 +++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index c6723b0..a664ddd 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -28,4 +28,15 @@ def get_instrument_fov(instrument): fov = 5 else: fov = 5 - return fov \ No newline at end of file + return fov + +def get_instrument_tev_range(array): + if array == 'lst': + erange = [0.03, 5] + elif array == 'mst': + erange = [1, 50] + elif array == 'sst': + erange = [5, 150] + else: + erange = [0.03, 150] + return erange diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 807fc6f..3fad414 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -16,9 +16,11 @@ def load_yaml_conf(yamlfile): CheckConfiguration(configuration=configuration) return configuration -def configure_simulator(simulator, configuration): +def configure_simulator_no_visibility(simulator, configuration): simulator.model = configuration['model'] simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed']) simulator.caldb = configuration['prod'] simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) + simulator.t = [0, configuration['duration']] + simulator.seed = configuration['seed'] -- GitLab From 056260e7776ec094ac81cda708f71bf06685f109 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:48:28 +0200 Subject: [PATCH 05/98] add test erange --- .../testing/test_astrort/test_utils/test_utils.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/astrort/testing/test_astrort/test_utils/test_utils.py b/astrort/testing/test_astrort/test_utils/test_utils.py index a78255b..59fefdb 100644 --- a/astrort/testing/test_astrort/test_utils/test_utils.py +++ b/astrort/testing/test_astrort/test_utils/test_utils.py @@ -7,7 +7,7 @@ # ***************************************************************************** import pytest -from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov +from astrort.utils.utils import * @pytest.mark.astrort_tmp_folder @pytest.mark.parametrize('samples', [3, 5, 8, 10]) @@ -37,4 +37,15 @@ def test_get_instrument_fov(array): assert fov == 5 return fov - +@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) +def test_get_instrument_tev_range(array): + erange = get_instrument_tev_range(array) + if array == 'lst': + assert erange == [0.03, 5] + elif array == 'mst': + assert erange == [1, 50] + elif array == 'sst': + assert erange == [5, 150] + else: + assert erange == [0.03, 150] + return erange \ No newline at end of file -- GitLab From 3adc7a0c6bd28ebb67859cca91d998f4b1dc80e0 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:49:04 +0200 Subject: [PATCH 06/98] typo --- .../testing/test_astrort/test_simulator/test_base_simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py b/astrort/testing/test_astrort/test_simulator/test_base_simulator.py index 0d184e9..3c3e26e 100644 --- a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py +++ b/astrort/testing/test_astrort/test_simulator/test_base_simulator.py @@ -10,7 +10,7 @@ import pytest from astrort.utils.wrap import load_yaml_conf from astrort.configure.check_configuration import CheckConfiguration -@pytest.mark.skip('to-do') +@pytest.mark.skip('#TODO') @pytest.mark.rtadeep_configuration class TestBaseSimulator: -- GitLab From 4a9e2ae5455b419ba4b1b5876fbf56b2557301be Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 17:53:50 +0200 Subject: [PATCH 07/98] def main --- astrort/simulator/base_simulator.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 41672c6..c9de431 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -7,16 +7,18 @@ # ***************************************************************************** import argparse -from rtasci.utils.RTACtoolsSimulation import RTACtoolsSimulation -from rtavis.utils.visibility import Visibility +from rtasci.rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf -from astrort.cfg.check_configuration import CheckConfiguration -parser = argparse.ArgumentParser(description='') -parser.add_argument('-cf', '--conf', type=str, required=True, help="Path of yaml configuration file") -args = parser.parse_args() +def main(): + parser = argparse.ArgumentParser(description='') + parser.add_argument('-cf', '--conf', type=str, required=True, help="Path of yaml configuration file") + args = parser.parse_args() -conf = load_yaml_conf(args.conf) + conf = load_yaml_conf(args.conf) + +if __name__ == '__main__': + main() -- GitLab From b97587b71b7d633936624f780e5e8001934a72f1 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 12 Sep 2023 18:15:00 +0200 Subject: [PATCH 08/98] simulate test --- astrort/configure/test.yml | 2 +- astrort/simulator/base_simulator.py | 16 ++++++++++++---- astrort/utils/wrap.py | 5 +++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 5a98e5a..7f0a48b 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -2,7 +2,7 @@ simulator: name: test array: lst irf: North_z60_0.5h_LST - prod: prod5 + prod: prod5-v0.1 pointing: random duration: 300 samples: 5 diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index c9de431..8447901 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -7,15 +7,23 @@ # ***************************************************************************** import argparse -from rtasci.rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf +from os import makedirs +from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility def main(): parser = argparse.ArgumentParser(description='') - parser.add_argument('-cf', '--conf', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") args = parser.parse_args() - conf = load_yaml_conf(args.conf) + configutation = load_yaml_conf(args.configuration) + makedirs(configutation['simulator']['output'], exist_ok=True) + + simulator = RTACtoolsSimulation() + for i in range(configutation['simulator']['samples']): + configutation['simulator']['seed'] += i + simulator = configure_simulator_no_visibility(simulator, configutation['simulator']) + simulator.run_simulation() if __name__ == '__main__': main() diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 3fad414..a036f79 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -7,6 +7,8 @@ # ***************************************************************************** import yaml +from os import makedirs +from os.path import dirname, abspath, join, basename from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov from astrort.configure.check_configuration import CheckConfiguration @@ -17,6 +19,8 @@ def load_yaml_conf(yamlfile): return configuration def configure_simulator_no_visibility(simulator, configuration): + if '$TEMPLATES$' in configuration['model']: + configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) simulator.model = configuration['model'] simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed']) simulator.caldb = configuration['prod'] @@ -24,3 +28,4 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] simulator.seed = configuration['seed'] + return simulator -- GitLab From 00d30945cb808195d982e5b93854c01003e35184 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 13:58:57 +0200 Subject: [PATCH 09/98] add logging --- astrort/configure/check_configuration.py | 9 +++- astrort/configure/logging.py | 28 +++++++++++ astrort/configure/test.yml | 5 +- .../test_check_configuration.py | 49 +++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 astrort/configure/logging.py create mode 100644 astrort/testing/test_configure/test_check_configuration.py diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index cf4f957..ae627f7 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -20,7 +20,7 @@ class CheckConfiguration(): return self def check_tags(self): - tags = ['simulator', 'visibility'] + tags = ['simulator', 'visibility', 'logging'] assert self.conf.keys() == tags return self @@ -43,3 +43,10 @@ class CheckConfiguration(): assert self.conf['visibility'].keys() == keys assert type(self.conf['visibility']['start_time']) == str return self + + def check_logging(self): + keys = ['level', 'logfile'] + assert self.conf['logging'].keys() == keys + assert (type(self.conf['logging']['level']) == str or type(self.conf['logging']['level']) == int) + assert type(self.conf['logging']['logfile']) == str + return self \ No newline at end of file diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py new file mode 100644 index 0000000..33368cf --- /dev/null +++ b/astrort/configure/logging.py @@ -0,0 +1,28 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import logging +from os import makedirs +from os.path import isdir, dirname + +def set_logger(level, filename=None): + log = logging.getLogger() + log.setLevel(level) + # console logger + formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s: %(message)s') + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + log.addHandler(consoleHandler) + # file logger + if filename is not None: + if not isdir(dirname(filename)): + makedirs(dirname(filename)) + fileHandler = logging.FileHandler(filename) + fileHandler.setFormatter(formatter) + log.addHandler(fileHandler) + return log \ No newline at end of file diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 7f0a48b..1515062 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -5,7 +5,7 @@ simulator: prod: prod5-v0.1 pointing: random duration: 300 - samples: 5 + samples: 1 seed: 1 model: $TEMPLATES$/crab.xml output: /data01/homes/dipiano/astroRT/testing/output_tmp @@ -13,3 +13,6 @@ simulator: visibility: start_time: '2030-01-01T00:00:00' +logging: + level: DEBUG + logfile: diff --git a/astrort/testing/test_configure/test_check_configuration.py b/astrort/testing/test_configure/test_check_configuration.py new file mode 100644 index 0000000..497ddd5 --- /dev/null +++ b/astrort/testing/test_configure/test_check_configuration.py @@ -0,0 +1,49 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from astrort.utils.wrap import load_yaml_conf +from astrort.configure.check_configuration import CheckConfiguration + +@pytest.mark.astrort_configuration +class TestCheckConfiguration: + + def test_check(self, astrort_configuration): + configuration = load_yaml_conf(astrort_configuration) + try: + CheckConfiguration(configuration=configuration).check() + except AssertionError as e: + type(e) == AssertionError + + def test_check_tags(self, astrort_configuration): + configuration = load_yaml_conf(astrort_configuration) + try: + CheckConfiguration(configuration=configuration).check_tags() + except AssertionError as e: + type(e) == AssertionError + + def test_check_simulator(self, astrort_configuration): + configuration = load_yaml_conf(astrort_configuration) + try: + CheckConfiguration(configuration=configuration).check_simulator() + except AssertionError as e: + type(e) == AssertionError + + def test_check_visibility(self, astrort_configuration): + configuration = load_yaml_conf(astrort_configuration) + try: + CheckConfiguration(configuration=configuration).check_visibility() + except AssertionError as e: + type(e) == AssertionError + + def test_check_logging(self, astrort_configuration): + configuration = load_yaml_conf(astrort_configuration) + try: + CheckConfiguration(configuration=configuration).check_logging() + except AssertionError as e: + type(e) == AssertionError \ No newline at end of file -- GitLab From c3552fb91e780cef6692975133409a222fe27dd2 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 13:59:11 +0200 Subject: [PATCH 10/98] moved --- .../test_check_configuration.py | 42 ------------------- .../test_simulator/test_base_simulator.py | 0 .../test_utils/test_utils.py | 8 ++-- .../test_utils/test_wrap.py | 0 4 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 astrort/testing/test_astrort/test_configure/test_check_configuration.py rename astrort/testing/{test_astrort => }/test_simulator/test_base_simulator.py (100%) rename astrort/testing/{test_astrort => }/test_utils/test_utils.py (85%) rename astrort/testing/{test_astrort => }/test_utils/test_wrap.py (100%) diff --git a/astrort/testing/test_astrort/test_configure/test_check_configuration.py b/astrort/testing/test_astrort/test_configure/test_check_configuration.py deleted file mode 100644 index 903a82d..0000000 --- a/astrort/testing/test_astrort/test_configure/test_check_configuration.py +++ /dev/null @@ -1,42 +0,0 @@ -# ***************************************************************************** -# Copyright (C) 2023 INAF -# This software is distributed under the terms of the BSD-3-Clause license -# -# Authors: -# Ambra Di Piano -# ***************************************************************************** - -import pytest -from astrort.utils.wrap import load_yaml_conf -from astrort.configure.check_configuration import CheckConfiguration - -@pytest.mark.astrort_configuration -class TestCheckConfiguration: - - def test_check(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check() - except AssertionError as e: - type(e) == AssertionError - - def test_check_tags(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_tags() - except AssertionError as e: - type(e) == AssertionError - - def test_check_simulator(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_simulator() - except AssertionError as e: - type(e) == AssertionError - - def test_check_visibility(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) - try: - CheckConfiguration(configuration=configuration).check_visibility() - except AssertionError as e: - type(e) == AssertionError diff --git a/astrort/testing/test_astrort/test_simulator/test_base_simulator.py b/astrort/testing/test_simulator/test_base_simulator.py similarity index 100% rename from astrort/testing/test_astrort/test_simulator/test_base_simulator.py rename to astrort/testing/test_simulator/test_base_simulator.py diff --git a/astrort/testing/test_astrort/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py similarity index 85% rename from astrort/testing/test_astrort/test_utils/test_utils.py rename to astrort/testing/test_utils/test_utils.py index 59fefdb..dd466ff 100644 --- a/astrort/testing/test_astrort/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -15,13 +15,13 @@ def test_seeds_to_string_formatter(samples, astrort_tmp_folder): name = seeds_to_string_formatter(samples, astrort_tmp_folder, name='test', seed=1) if samples <= 3: - assert name == f"{astrort_tmp_folder}/test_001" + assert name == f"{astrort_tmp_folder}/test_001.fits" elif samples <= 5: - assert name == f"{astrort_tmp_folder}/test_00001" + assert name == f"{astrort_tmp_folder}/test_00001.fits" elif samples <= 8: - assert name == f"{astrort_tmp_folder}/test_00000001" + assert name == f"{astrort_tmp_folder}/test_00000001.fits" else: - assert name == f"{astrort_tmp_folder}/test_1" + assert name == f"{astrort_tmp_folder}/test_1.fits" @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) def test_get_instrument_fov(array): diff --git a/astrort/testing/test_astrort/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py similarity index 100% rename from astrort/testing/test_astrort/test_utils/test_wrap.py rename to astrort/testing/test_utils/test_wrap.py -- GitLab From 5e7bd53c2f711c18db693b465278e28c4785569a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 13:59:23 +0200 Subject: [PATCH 11/98] fix extension --- astrort/utils/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index a664ddd..c525911 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -10,13 +10,13 @@ from os.path import join def seeds_to_string_formatter(samples, output, name, seed): if samples <= 3: - name = join(output, f"{name}_{seed:03d}") + name = join(output, f"{name}_{seed:03d}.fits") elif samples <= 5: - name = join(output, f"{name}_{seed:05d}") + name = join(output, f"{name}_{seed:05d}.fits") elif samples <= 8: - name = join(output, f"{name}_{seed:08d}") + name = join(output, f"{name}_{seed:08d}.fits") else: - name = join(output, f"{name}_{seed}") + name = join(output, f"{name}_{seed}.fits") return name def get_instrument_fov(instrument): -- GitLab From 7c2e9cdfa86594b012a9b854694c7d2d10b5b02b Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 13:59:35 +0200 Subject: [PATCH 12/98] simulator base --- astrort/simulator/base_simulator.py | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 8447901..6a366d0 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,23 +10,42 @@ import argparse from os import makedirs from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility +from astrort.configure.logging import set_logger -def main(): +def base_simulator(configuration_file): + configuration = load_yaml_conf(configuration_file) + log = set_logger(configuration['logging']['level']) + # create output dir + log.info(f"Creating {configuration['simulator']['output']}") + makedirs(configuration['simulator']['output'], exist_ok=True) + + # start simulations + print(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") + for i in range(configuration['simulator']['samples']): + simulator = RTACtoolsSimulation() + configuration['simulator']['seed'] += i + simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) + simulator.run_simulation() + print(f"Simulation (seed = {configuration['simulator']['seed']}) complete") + del simulator + # end simulations + print(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} ") + + +def slurm_submission(configuration_file): + return + +if __name__ == '__main__': parser = argparse.ArgumentParser(description='') parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) args = parser.parse_args() - configutation = load_yaml_conf(args.configuration) - makedirs(configutation['simulator']['output'], exist_ok=True) - - simulator = RTACtoolsSimulation() - for i in range(configutation['simulator']['samples']): - configutation['simulator']['seed'] += i - simulator = configure_simulator_no_visibility(simulator, configutation['simulator']) - simulator.run_simulation() -if __name__ == '__main__': - main() + if args.nodes == 0: + base_simulator(args.configuration) + else: + slurm_submission(args.configuration) -- GitLab From 0832bb848a06520babddef62657c666eb09b45f4 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:39:06 +0200 Subject: [PATCH 13/98] update --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b3ff366..e1937c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ # Patterns .* *.png +*.yml # Folders tmp/ +tmp_*/ analysis_archive/ # Files @@ -15,6 +17,7 @@ rtasci/rtasci/cfg/leo_bkg.yml !rtamock/execute_run.sh !environment.yml !rtavis/rtavis/lib/*py +!astrort/configure/test.yml # Byte-compiled / optimized / DLL files __pycache__/ -- GitLab From 3768223e6697b899348db522c4fa330a4365177a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:39:16 +0200 Subject: [PATCH 14/98] get log level --- astrort/configure/logging.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py index 33368cf..f5c12fb 100644 --- a/astrort/configure/logging.py +++ b/astrort/configure/logging.py @@ -25,4 +25,21 @@ def set_logger(level, filename=None): fileHandler = logging.FileHandler(filename) fileHandler.setFormatter(formatter) log.addHandler(fileHandler) - return log \ No newline at end of file + return log + +def get_log_level(level): + if level in ['DEBUG', 'debug', 10]: + level = logging.DEBUG + elif level in ['INFO', 'info', 20]: + level = logging.INFO + elif level in ['WARN', 'WARNING', 'warn', 'warning', 30]: + level = logging.WARN + elif level in ['ERROR', 'error', 40]: + level = logging.ERROR + elif level in ['CRITICAL', 'critical', 50]: + level = logging.CRITICAL + else: + level = logging.NOTSET + return level + + return \ No newline at end of file -- GitLab From 0a543db47188794b4a0ead6dd74b8b454d2c628f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:39:26 +0200 Subject: [PATCH 15/98] unit test conf --- astrort/configure/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 1515062..3737d3e 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -4,15 +4,15 @@ simulator: irf: North_z60_0.5h_LST prod: prod5-v0.1 pointing: random - duration: 300 - samples: 1 + duration: 10 + samples: 5 seed: 1 model: $TEMPLATES$/crab.xml - output: /data01/homes/dipiano/astroRT/testing/output_tmp + output: /data01/homes/dipiano/astroRT/astrort/testing/tmp visibility: start_time: '2030-01-01T00:00:00' logging: - level: DEBUG + level: CRITICAL logfile: -- GitLab From 1b55b2a983e04c252c26d8a3bdc5a259d433508f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:39:40 +0200 Subject: [PATCH 16/98] fix seed and add logger --- astrort/simulator/base_simulator.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 6a366d0..539a3db 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,26 +10,25 @@ import argparse from os import makedirs from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility -from astrort.configure.logging import set_logger +from astrort.configure.logging import set_logger, get_log_level def base_simulator(configuration_file): configuration = load_yaml_conf(configuration_file) - log = set_logger(configuration['logging']['level']) + log = set_logger(get_log_level(configuration['logging']['level'])) # create output dir log.info(f"Creating {configuration['simulator']['output']}") makedirs(configuration['simulator']['output'], exist_ok=True) - # start simulations print(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") for i in range(configuration['simulator']['samples']): simulator = RTACtoolsSimulation() - configuration['simulator']['seed'] += i simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) simulator.run_simulation() print(f"Simulation (seed = {configuration['simulator']['seed']}) complete") + configuration['simulator']['seed'] += 1 del simulator # end simulations - print(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} ") + print(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") def slurm_submission(configuration_file): -- GitLab From 54118ee036a962032cfa0e844c669440a0ff7134 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:39:59 +0200 Subject: [PATCH 17/98] rename fixture test_conf_file --- astrort/testing/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/testing/conftest.py b/astrort/testing/conftest.py index 4edcb34..6e12196 100644 --- a/astrort/testing/conftest.py +++ b/astrort/testing/conftest.py @@ -12,7 +12,7 @@ from os.path import join, dirname, abspath from os import makedirs @pytest.fixture(scope='function') -def astrort_configuration(): +def test_conf_file(): return join(dirname(abspath(astrort.__file__)), 'configure', 'test.yml') @pytest.fixture(scope='function') -- GitLab From 9fcc2dc8c246f6fef39e2a59e7ba52528190ab3d Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:40:13 +0200 Subject: [PATCH 18/98] rename fixture test_conf_file --- astrort/testing/pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/testing/pytest.ini b/astrort/testing/pytest.ini index c37ccaa..8f8829b 100644 --- a/astrort/testing/pytest.ini +++ b/astrort/testing/pytest.ini @@ -1,4 +1,4 @@ [pytest] markers = - astrort_configuration: test configurtion file + test_conf_file: test configurtion file astrort_tmp_folder: test tmp output folder \ No newline at end of file -- GitLab From 7cd7fd0984e71db2b17bc23adeaac1bf82fb6d32 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:40:38 +0200 Subject: [PATCH 19/98] add test_base_simulator for no slurm use --- .../test_simulator/test_base_simulator.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/astrort/testing/test_simulator/test_base_simulator.py b/astrort/testing/test_simulator/test_base_simulator.py index 3c3e26e..4c35fcb 100644 --- a/astrort/testing/test_simulator/test_base_simulator.py +++ b/astrort/testing/test_simulator/test_base_simulator.py @@ -7,17 +7,26 @@ # ***************************************************************************** import pytest +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator from astrort.utils.wrap import load_yaml_conf -from astrort.configure.check_configuration import CheckConfiguration -@pytest.mark.skip('#TODO') -@pytest.mark.rtadeep_configuration -class TestBaseSimulator: +@pytest.mark.test_conf_file +def test_base_simulator(test_conf_file): - @pytest.mark.skip('to-do') - def test_base_simulator(self, rtadeep_configuration): + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['simulator']['output']) - # get configuration - configuration = load_yaml_conf(rtadeep_configuration) + # run simulator + base_simulator(test_conf_file) - + # check output + expected_simulations = conf['simulator']['samples'] + found_simulations = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.fits' in f and conf['simulator']['name'] in f]) + assert found_simulations == expected_simulations, f"Expected {expected_simulations} simulations, found {found_simulations}" + + + -- GitLab From 37df3157a77c2cd04e11f931539b00e802262e30 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:40:51 +0200 Subject: [PATCH 20/98] rename fixture test_conf_file --- .../test_check_configuration.py | 22 +++++++++---------- astrort/testing/test_utils/test_utils.py | 6 ++--- astrort/testing/test_utils/test_wrap.py | 6 ++--- astrort/utils/utils.py | 6 ++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/astrort/testing/test_configure/test_check_configuration.py b/astrort/testing/test_configure/test_check_configuration.py index 497ddd5..15a787f 100644 --- a/astrort/testing/test_configure/test_check_configuration.py +++ b/astrort/testing/test_configure/test_check_configuration.py @@ -10,39 +10,39 @@ import pytest from astrort.utils.wrap import load_yaml_conf from astrort.configure.check_configuration import CheckConfiguration -@pytest.mark.astrort_configuration +@pytest.mark.test_conf_file class TestCheckConfiguration: - def test_check(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) + def test_check(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check() except AssertionError as e: type(e) == AssertionError - def test_check_tags(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) + def test_check_tags(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_tags() except AssertionError as e: type(e) == AssertionError - def test_check_simulator(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) + def test_check_simulator(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_simulator() except AssertionError as e: type(e) == AssertionError - def test_check_visibility(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) + def test_check_visibility(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_visibility() except AssertionError as e: type(e) == AssertionError - def test_check_logging(self, astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) + def test_check_logging(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_logging() except AssertionError as e: diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index dd466ff..cdc3e46 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -14,11 +14,11 @@ from astrort.utils.utils import * def test_seeds_to_string_formatter(samples, astrort_tmp_folder): name = seeds_to_string_formatter(samples, astrort_tmp_folder, name='test', seed=1) - if samples <= 3: + if samples <= 1e3: assert name == f"{astrort_tmp_folder}/test_001.fits" - elif samples <= 5: + elif samples <= 1e5: assert name == f"{astrort_tmp_folder}/test_00001.fits" - elif samples <= 8: + elif samples <= 1e8: assert name == f"{astrort_tmp_folder}/test_00000001.fits" else: assert name == f"{astrort_tmp_folder}/test_1.fits" diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 04cb39b..54f28de 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -9,7 +9,7 @@ import pytest from astrort.utils.wrap import load_yaml_conf -@pytest.mark.astrort_configuration -def test_load_yaml_conf(astrort_configuration): - configuration = load_yaml_conf(astrort_configuration) +@pytest.mark.test_conf_file +def test_load_yaml_conf(test_conf_file): + configuration = load_yaml_conf(test_conf_file) assert type(configuration) == dict \ No newline at end of file diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index c525911..c4a8fd5 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -9,11 +9,11 @@ from os.path import join def seeds_to_string_formatter(samples, output, name, seed): - if samples <= 3: + if samples <= 1e3: name = join(output, f"{name}_{seed:03d}.fits") - elif samples <= 5: + elif samples <= 1e5: name = join(output, f"{name}_{seed:05d}.fits") - elif samples <= 8: + elif samples <= 1e8: name = join(output, f"{name}_{seed:08d}.fits") else: name = join(output, f"{name}_{seed}.fits") -- GitLab From 4743998e5900580b8e213844462edb271bd768b7 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 14:59:16 +0200 Subject: [PATCH 21/98] rm unused import --- astrort/utils/wrap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index a036f79..6354bfe 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -7,7 +7,6 @@ # ***************************************************************************** import yaml -from os import makedirs from os.path import dirname, abspath, join, basename from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov from astrort.configure.check_configuration import CheckConfiguration -- GitLab From ae35622ed43194a73d7d61f93829d9d1e4c82de5 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:45:53 +0200 Subject: [PATCH 22/98] rename test_tmp_folder fixture --- astrort/testing/conftest.py | 2 +- astrort/testing/pytest.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astrort/testing/conftest.py b/astrort/testing/conftest.py index 6e12196..115a74f 100644 --- a/astrort/testing/conftest.py +++ b/astrort/testing/conftest.py @@ -16,6 +16,6 @@ def test_conf_file(): return join(dirname(abspath(astrort.__file__)), 'configure', 'test.yml') @pytest.fixture(scope='function') -def astrort_tmp_folder(): +def test_tmp_folder(): makedirs(join(dirname(abspath(astrort.__file__)), 'testing', 'tmp'), exist_ok=True) return join(dirname(abspath(astrort.__file__)), 'testing', 'tmp') diff --git a/astrort/testing/pytest.ini b/astrort/testing/pytest.ini index 8f8829b..e79d0a6 100644 --- a/astrort/testing/pytest.ini +++ b/astrort/testing/pytest.ini @@ -1,4 +1,4 @@ [pytest] markers = test_conf_file: test configurtion file - astrort_tmp_folder: test tmp output folder \ No newline at end of file + test_tmp_folder: test tmp output folder \ No newline at end of file -- GitLab From 3f1d01931e6097aab9eb97391f164aa4e1831d57 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:46:21 +0200 Subject: [PATCH 23/98] test log --- .../testing/test_configure/test_logging.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 astrort/testing/test_configure/test_logging.py diff --git a/astrort/testing/test_configure/test_logging.py b/astrort/testing/test_configure/test_logging.py new file mode 100644 index 0000000..61a600b --- /dev/null +++ b/astrort/testing/test_configure/test_logging.py @@ -0,0 +1,26 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +import logging +from os.path import join, isfile +from astrort.configure.logging import get_log_level, set_logger + +def test_set_logger(): + assert get_log_level('DEBUG') == logging.DEBUG + assert get_log_level('INFO') == logging.INFO + assert get_log_level('WARNING') == logging.WARNING + assert get_log_level('ERROR') == logging.ERROR + assert get_log_level('CRITICAL') == logging.CRITICAL + +@pytest.mark.test_tmp_folder +@pytest.mark.test_conf_file +def test_set_logger(test_tmp_folder): + log = set_logger(logging.DEBUG, join(test_tmp_folder, 'test_set_logger.log')) + log.debug('TEST') + assert isfile(join(test_tmp_folder, 'test_set_logger.log')) is True -- GitLab From e1433b37585d206087cda0a5e6d6931393d3e51b Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:46:31 +0200 Subject: [PATCH 24/98] update formatter --- astrort/configure/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py index f5c12fb..975f725 100644 --- a/astrort/configure/logging.py +++ b/astrort/configure/logging.py @@ -14,7 +14,7 @@ def set_logger(level, filename=None): log = logging.getLogger() log.setLevel(level) # console logger - formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s: %(message)s') + formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s') consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(formatter) log.addHandler(consoleHandler) -- GitLab From 9cb0bb694bd9f2ac94fdf7e345deb8f40bd77c83 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:46:44 +0200 Subject: [PATCH 25/98] rename fixture --- astrort/testing/test_utils/test_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index cdc3e46..4663962 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -9,19 +9,19 @@ import pytest from astrort.utils.utils import * -@pytest.mark.astrort_tmp_folder +@pytest.mark.test_tmp_folder @pytest.mark.parametrize('samples', [3, 5, 8, 10]) -def test_seeds_to_string_formatter(samples, astrort_tmp_folder): - name = seeds_to_string_formatter(samples, astrort_tmp_folder, name='test', seed=1) +def test_seeds_to_string_formatter(samples, test_tmp_folder): + name = seeds_to_string_formatter(samples, test_tmp_folder, name='test', seed=1) if samples <= 1e3: - assert name == f"{astrort_tmp_folder}/test_001.fits" + assert name == f"{test_tmp_folder}/test_001.fits" elif samples <= 1e5: - assert name == f"{astrort_tmp_folder}/test_00001.fits" + assert name == f"{test_tmp_folder}/test_00001.fits" elif samples <= 1e8: - assert name == f"{astrort_tmp_folder}/test_00000001.fits" + assert name == f"{test_tmp_folder}/test_00000001.fits" else: - assert name == f"{astrort_tmp_folder}/test_1.fits" + assert name == f"{test_tmp_folder}/test_1.fits" @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) def test_get_instrument_fov(array): -- GitLab From 8a922b0b1a81ae1c94958df1d177cca09f04c5b6 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:50:00 +0200 Subject: [PATCH 26/98] update log level --- astrort/testing/test_configure/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/testing/test_configure/test_logging.py b/astrort/testing/test_configure/test_logging.py index 61a600b..30a6c0d 100644 --- a/astrort/testing/test_configure/test_logging.py +++ b/astrort/testing/test_configure/test_logging.py @@ -21,6 +21,6 @@ def test_set_logger(): @pytest.mark.test_tmp_folder @pytest.mark.test_conf_file def test_set_logger(test_tmp_folder): - log = set_logger(logging.DEBUG, join(test_tmp_folder, 'test_set_logger.log')) + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) log.debug('TEST') assert isfile(join(test_tmp_folder, 'test_set_logger.log')) is True -- GitLab From f52072a41c1b39c7726efba187d61b91224265c5 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 15:50:13 +0200 Subject: [PATCH 27/98] fix print now log --- astrort/simulator/base_simulator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 539a3db..91f6eb7 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -19,16 +19,16 @@ def base_simulator(configuration_file): log.info(f"Creating {configuration['simulator']['output']}") makedirs(configuration['simulator']['output'], exist_ok=True) # start simulations - print(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") + log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") for i in range(configuration['simulator']['samples']): simulator = RTACtoolsSimulation() simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) simulator.run_simulation() - print(f"Simulation (seed = {configuration['simulator']['seed']}) complete") + log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete") configuration['simulator']['seed'] += 1 del simulator # end simulations - print(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") + log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") def slurm_submission(configuration_file): -- GitLab From 612f8d2aad38974ab847fe5ef8fff097866c6886 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 16:20:52 +0200 Subject: [PATCH 28/98] update env --- environment.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/environment.yml b/environment.yml index 9bc21a3..0fc03d5 100644 --- a/environment.yml +++ b/environment.yml @@ -164,7 +164,3 @@ dependencies: - zipp=3.11.0=py38h06a4308_0 - zlib=1.2.13=h5eee18b_0 - zstd=1.5.5=hc292b87_0 - - pip: - - astrort==0.0.0 - - rtavis==0.0.0 -prefix: /data01/homes/dipiano/.conda/envs/astrort -- GitLab From 5427a6a948c0fbf83bb23fbd4ad95a781c3b6dcf Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 13 Sep 2023 16:34:32 +0200 Subject: [PATCH 29/98] rearrange channels --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 0fc03d5..2cae062 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,9 @@ name: astrort channels: - - anaconda - - defaults - conda-forge - cta-observatory + - anaconda + - defaults dependencies: - _libgcc_mutex=0.1=conda_forge - _openmp_mutex=4.5=2_kmp_llvm -- GitLab From cf6416d90a3d9be9707a0855b999967584075165 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 14 Sep 2023 14:03:02 +0200 Subject: [PATCH 30/98] update env with no builds --- environment.yml | 319 ++++++++++++++++++++++++------------------------ 1 file changed, 160 insertions(+), 159 deletions(-) diff --git a/environment.yml b/environment.yml index 2cae062..7518bf8 100644 --- a/environment.yml +++ b/environment.yml @@ -5,162 +5,163 @@ channels: - anaconda - defaults dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_kmp_llvm - - astropy=5.1=py38h7deecbd_0 - - asttokens=2.0.5=pyhd3eb1b0_0 - - attrs=22.1.0=py38h06a4308_0 - - backcall=0.2.0=pyhd3eb1b0_0 - - blas=1.0=mkl - - brotli=1.0.9=h5eee18b_7 - - brotli-bin=1.0.9=h5eee18b_7 - - c-ares=1.19.1=h5eee18b_0 - - ca-certificates=2023.01.10=h06a4308_0 - - cfitsio=3.430=hd130d23_1 - - comm=0.1.2=py38h06a4308_0 - - contourpy=1.0.5=py38hdb19cb5_0 - - ctools=2.0.0=py38_1 - - curl=8.2.1=hdbd6064_0 - - cycler=0.11.0=pyhd3eb1b0_0 - - cyrus-sasl=2.1.28=h52b45da_1 - - dbus=1.13.18=hb2f20db_0 - - debugpy=1.6.7=py38h6a678d5_0 - - decorator=5.1.1=pyhd3eb1b0_0 - - exceptiongroup=1.0.4=py38h06a4308_0 - - executing=0.8.3=pyhd3eb1b0_0 - - expat=2.5.0=h6a678d5_0 - - fontconfig=2.14.1=h4c34cd2_2 - - fonttools=4.25.0=pyhd3eb1b0_0 - - freetype=2.12.1=h4a9f257_0 - - gammalib=2.0.0=py38_1 - - giflib=5.2.1=h5eee18b_3 - - glib=2.69.1=he621ea3_2 - - gst-plugins-base=1.14.1=h6a678d5_1 - - gstreamer=1.14.1=h5eee18b_1 - - healpy=1.13.0=py38hf476568_1 - - icu=58.2=he6710b0_3 - - importlib-metadata=6.0.0=py38h06a4308_0 - - importlib_metadata=6.0.0=hd3eb1b0_0 - - importlib_resources=5.2.0=pyhd3eb1b0_1 - - iniconfig=1.1.1=pyhd3eb1b0_0 - - intel-openmp=2023.1.0=hdb19cb5_46305 - - ipykernel=6.25.0=py38h2f386ee_0 - - ipython=8.12.2=py38h06a4308_0 - - jedi=0.18.1=py38h06a4308_1 - - jpeg=9e=h5eee18b_1 - - jupyter_client=8.1.0=py38h06a4308_0 - - jupyter_core=5.3.0=py38h06a4308_0 - - kiwisolver=1.4.4=py38h6a678d5_0 - - krb5=1.20.1=h143b758_1 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libblas=3.9.0=1_h6e990d7_netlib - - libbrotlicommon=1.0.9=h5eee18b_7 - - libbrotlidec=1.0.9=h5eee18b_7 - - libbrotlienc=1.0.9=h5eee18b_7 - - libcblas=3.9.0=3_h893e4fe_netlib - - libclang=14.0.6=default_hc6dbbc7_1 - - libclang13=14.0.6=default_he11475f_1 - - libcups=2.4.2=h2d74bed_1 - - libcurl=8.2.1=h251f7ec_0 - - libdeflate=1.17=h5eee18b_0 - - libedit=3.1.20221030=h5eee18b_0 - - libev=4.33=h7f8727e_1 - - libevent=2.1.12=hdbd6064_1 - - libffi=3.4.4=h6a678d5_0 - - libgcc-ng=13.1.0=he5830b7_0 - - libgfortran=3.0.0=1 - - libgfortran-ng=7.5.0=ha8ba4b0_17 - - libgfortran4=7.5.0=ha8ba4b0_17 - - liblapack=3.9.0=3_h893e4fe_netlib - - libllvm14=14.0.6=hdb19cb5_3 - - libnghttp2=1.52.0=h2d74bed_1 - - libpng=1.6.39=h5eee18b_0 - - libpq=12.15=hdbd6064_1 - - libsodium=1.0.18=h7b6447c_0 - - libssh2=1.10.0=hdbd6064_2 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtiff=4.5.1=h6a678d5_0 - - libuuid=1.41.5=h5eee18b_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - libxcb=1.15=h7f8727e_0 - - libxkbcommon=1.0.1=h5eee18b_1 - - libxml2=2.10.4=hcbfbd50_0 - - libxslt=1.1.37=h2085143_0 - - llvm-openmp=14.0.6=h9e868ea_0 - - lxml=4.9.2=py38h5eee18b_0 - - lz4-c=1.9.4=h6a678d5_0 - - matplotlib=3.7.2=py38h06a4308_0 - - matplotlib-base=3.7.2=py38h1128e8f_0 - - matplotlib-inline=0.1.6=py38h06a4308_0 - - mkl=2023.1.0=h213fc3f_46343 - - mkl-service=2.4.0=py38h5eee18b_1 - - mkl_fft=1.3.6=py38h417a72b_1 - - mkl_random=1.2.2=py38h417a72b_1 - - munkres=1.1.4=py_0 - - mysql=5.7.24=h721c034_2 - - ncurses=6.4=h6a678d5_0 - - nest-asyncio=1.5.6=py38h06a4308_0 - - nspr=4.35=h6a678d5_0 - - nss=3.89.1=h6a678d5_0 - - numpy=1.24.3=py38hf6e8229_1 - - numpy-base=1.24.3=py38h060ed82_1 - - openssl=3.0.10=h7f8727e_2 - - packaging=23.1=py38h06a4308_0 - - pandas=1.3.5=py38h43a58ef_0 - - parso=0.8.3=pyhd3eb1b0_0 - - pcre=8.45=h295c915_0 - - pexpect=4.8.0=pyhd3eb1b0_3 - - pickleshare=0.7.5=pyhd3eb1b0_1003 - - pillow=9.4.0=py38h6a678d5_0 - - pip=23.2.1=py38h06a4308_0 - - platformdirs=3.10.0=py38h06a4308_0 - - pluggy=1.0.0=py38h06a4308_1 - - ply=3.11=py38_0 - - prompt-toolkit=3.0.36=py38h06a4308_0 - - psutil=5.9.0=py38h5eee18b_0 - - ptyprocess=0.7.0=pyhd3eb1b0_2 - - pure_eval=0.2.2=pyhd3eb1b0_0 - - py=1.11.0=pyhd3eb1b0_0 - - pyerfa=2.0.0=py38h27cfd23_0 - - pygments=2.15.1=py38h06a4308_1 - - pyparsing=3.0.9=py38h06a4308_0 - - pyqt=5.15.7=py38h6a678d5_1 - - pyqt5-sip=12.11.0=py38h6a678d5_1 - - pytest=7.1.2=py38h06a4308_0 - - pytest-runner=6.0.0=py38h06a4308_0 - - python=3.8.17=h955ad1f_0 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python_abi=3.8=2_cp38 - - pytz=2022.7=py38h06a4308_0 - - pyyaml=6.0=py38h5eee18b_1 - - pyzmq=25.1.0=py38h6a678d5_0 - - qt-main=5.15.2=h7358343_9 - - qt-webengine=5.15.9=h9ab4d14_7 - - qtwebkit=5.212=h3fafdc1_5 - - readline=8.2=h5eee18b_0 - - regions=0.7=py38h26c90d9_2 - - scipy=1.5.3=py38h828c644_0 - - seaborn=0.12.2=py38h06a4308_0 - - setuptools=59.8.0=py38h578d9bd_1 - - sip=6.6.2=py38h6a678d5_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.2=h5eee18b_0 - - stack_data=0.2.0=pyhd3eb1b0_0 - - tbb=2021.8.0=hdb19cb5_0 - - tk=8.6.12=h1ccaba5_0 - - toml=0.10.2=pyhd3eb1b0_0 - - tomli=2.0.1=py38h06a4308_0 - - tornado=6.3.2=py38h5eee18b_0 - - traitlets=5.7.1=py38h06a4308_0 - - typing_extensions=4.7.1=py38h06a4308_0 - - wcwidth=0.2.5=pyhd3eb1b0_0 - - wheel=0.38.4=py38h06a4308_0 - - xz=5.4.2=h5eee18b_0 - - yaml=0.2.5=h7b6447c_0 - - zeromq=4.3.4=h2531618_0 - - zipp=3.11.0=py38h06a4308_0 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.5=hc292b87_0 + - _libgcc_mutex=0.1 + - _openmp_mutex=4.5 + - astropy=5.1 + - asttokens=2.0.5 + - attrs=22.1.0 + - backcall=0.2.0 + - blas=1.0 + - brotli=1.0.9 + - brotli-bin=1.0.9 + - c-ares=1.19.1 + - ca-certificates=2023.01.10 + - cfitsio=3.430 + - comm=0.1.2 + - contourpy=1.0.5 + - ctools=2.0.0 + - curl=8.2.1 + - cycler=0.11.0 + - cyrus-sasl=2.1.28 + - dbus=1.13.18 + - debugpy=1.6.7 + - decorator=5.1.1 + - exceptiongroup=1.0.4 + - executing=0.8.3 + - expat=2.5.0 + - fontconfig=2.14.1 + - fonttools=4.25.0 + - freetype=2.12.1 + - gammalib=2.0.0 + - giflib=5.2.1 + - glib=2.69.1 + - gst-plugins-base=1.14.1 + - gstreamer=1.14.1 + - healpy=1.13.0 + - icu=58.2 + - importlib-metadata=6.0.0 + - importlib_metadata=6.0.0 + - importlib_resources=5.2.0 + - iniconfig=1.1.1 + - intel-openmp=2023.1.0 + - ipykernel=6.25.0 + - ipython=8.12.2 + - jedi=0.18.1 + - jpeg=9e + - jupyter_client=8.1.0 + - jupyter_core=5.3.0 + - kiwisolver=1.4.4 + - krb5=1.20.1 + - lcms2=2.12 + - ld_impl_linux-64=2.38 + - lerc=3.0 + - libblas=3.9.0 + - libbrotlicommon=1.0.9 + - libbrotlidec=1.0.9 + - libbrotlienc=1.0.9 + - libcblas=3.9.0 + - libclang=14.0.6 + - libclang13=14.0.6 + - libcups=2.4.2 + - libcurl=8.2.1 + - libdeflate=1.17 + - libedit=3.1.20221030 + - libev=4.33 + - libevent=2.1.12 + - libffi=3.4.4 + - libgcc-ng=13.1.0 + - libgfortran=3.0.0 + - libgfortran-ng=7.5.0 + - libgfortran4=7.5.0 + - liblapack=3.9.0 + - libllvm14=14.0.6 + - libnghttp2=1.52.0 + - libpng=1.6.39 + - libpq=12.15 + - libsodium=1.0.18 + - libssh2=1.10.0 + - libstdcxx-ng=11.2.0 + - libtiff=4.5.1 + - libuuid=1.41.5 + - libwebp=1.2.4 + - libwebp-base=1.2.4 + - libxcb=1.15 + - libxkbcommon=1.0.1 + - libxml2=2.10.4 + - libxslt=1.1.37 + - llvm-openmp=14.0.6 + - lxml=4.9.2 + - lz4-c=1.9.4 + - matplotlib=3.7.2 + - matplotlib-base=3.7.2 + - matplotlib-inline=0.1.6 + - mkl=2023.1.0 + - mkl-service=2.4.0 + - mkl_fft=1.3.6 + - mkl_random=1.2.2 + - munkres=1.1.4 + - mysql=5.7.24 + - ncurses=6.4 + - nest-asyncio=1.5.6 + - nspr=4.35 + - nss=3.89.1 + - numpy=1.24.3 + - numpy-base=1.24.3 + - openssl=3.0.10 + - packaging=23.1 + - pandas=1.3.5 + - parso=0.8.3 + - pcre=8.45 + - pexpect=4.8.0 + - pickleshare=0.7.5 + - pillow=9.4.0 + - pip=23.2.1 + - platformdirs=3.10.0 + - pluggy=1.0.0 + - ply=3.11 + - prompt-toolkit=3.0.36 + - psutil=5.9.0 + - ptyprocess=0.7.0 + - pure_eval=0.2.2 + - py=1.11.0 + - pyerfa=2.0.0 + - pygments=2.15.1 + - pyparsing=3.0.9 + - pyqt=5.15.7 + - pyqt5-sip=12.11.0 + - pytest=7.1.2 + - pytest-runner=6.0.0 + - python=3.8.17 + - python-dateutil=2.8.2 + - python_abi=3.8 + - pytz=2022.7 + - pyyaml=6.0 + - pyzmq=25.1.0 + - qt-main=5.15.2 + - qt-webengine=5.15.9 + - qtwebkit=5.212 + - readline=8.2 + - regions=0.7 + - scipy=1.5.3 + - seaborn=0.12.2 + - setuptools=59.8.0 + - sip=6.6.2 + - six=1.16.0 + - sqlite=3.41.2 + - stack_data=0.2.0 + - tbb=2021.8.0 + - tk=8.6.12 + - toml=0.10.2 + - tomli=2.0.1 + - tornado=6.3.2 + - traitlets=5.7.1 + - typing_extensions=4.7.1 + - wcwidth=0.2.5 + - wheel=0.38.4 + - xz=5.4.2 + - yaml=0.2.5 + - zeromq=4.3.4 + - zipp=3.11.0 + - zlib=1.2.13 + - zstd=1.5.5 + -- GitLab From 6ef509c5ff7f3865af025081093b9b44f3509a6e Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 13:14:34 +0200 Subject: [PATCH 31/98] simplify env --- environment.yml | 174 +++--------------------------------------------- 1 file changed, 9 insertions(+), 165 deletions(-) diff --git a/environment.yml b/environment.yml index 7518bf8..5c197d4 100644 --- a/environment.yml +++ b/environment.yml @@ -1,167 +1,11 @@ name: astrort -channels: - - conda-forge - - cta-observatory - - anaconda - - defaults dependencies: - - _libgcc_mutex=0.1 - - _openmp_mutex=4.5 - - astropy=5.1 - - asttokens=2.0.5 - - attrs=22.1.0 - - backcall=0.2.0 - - blas=1.0 - - brotli=1.0.9 - - brotli-bin=1.0.9 - - c-ares=1.19.1 - - ca-certificates=2023.01.10 - - cfitsio=3.430 - - comm=0.1.2 - - contourpy=1.0.5 - - ctools=2.0.0 - - curl=8.2.1 - - cycler=0.11.0 - - cyrus-sasl=2.1.28 - - dbus=1.13.18 - - debugpy=1.6.7 - - decorator=5.1.1 - - exceptiongroup=1.0.4 - - executing=0.8.3 - - expat=2.5.0 - - fontconfig=2.14.1 - - fonttools=4.25.0 - - freetype=2.12.1 - - gammalib=2.0.0 - - giflib=5.2.1 - - glib=2.69.1 - - gst-plugins-base=1.14.1 - - gstreamer=1.14.1 - - healpy=1.13.0 - - icu=58.2 - - importlib-metadata=6.0.0 - - importlib_metadata=6.0.0 - - importlib_resources=5.2.0 - - iniconfig=1.1.1 - - intel-openmp=2023.1.0 - - ipykernel=6.25.0 - - ipython=8.12.2 - - jedi=0.18.1 - - jpeg=9e - - jupyter_client=8.1.0 - - jupyter_core=5.3.0 - - kiwisolver=1.4.4 - - krb5=1.20.1 - - lcms2=2.12 - - ld_impl_linux-64=2.38 - - lerc=3.0 - - libblas=3.9.0 - - libbrotlicommon=1.0.9 - - libbrotlidec=1.0.9 - - libbrotlienc=1.0.9 - - libcblas=3.9.0 - - libclang=14.0.6 - - libclang13=14.0.6 - - libcups=2.4.2 - - libcurl=8.2.1 - - libdeflate=1.17 - - libedit=3.1.20221030 - - libev=4.33 - - libevent=2.1.12 - - libffi=3.4.4 - - libgcc-ng=13.1.0 - - libgfortran=3.0.0 - - libgfortran-ng=7.5.0 - - libgfortran4=7.5.0 - - liblapack=3.9.0 - - libllvm14=14.0.6 - - libnghttp2=1.52.0 - - libpng=1.6.39 - - libpq=12.15 - - libsodium=1.0.18 - - libssh2=1.10.0 - - libstdcxx-ng=11.2.0 - - libtiff=4.5.1 - - libuuid=1.41.5 - - libwebp=1.2.4 - - libwebp-base=1.2.4 - - libxcb=1.15 - - libxkbcommon=1.0.1 - - libxml2=2.10.4 - - libxslt=1.1.37 - - llvm-openmp=14.0.6 - - lxml=4.9.2 - - lz4-c=1.9.4 - - matplotlib=3.7.2 - - matplotlib-base=3.7.2 - - matplotlib-inline=0.1.6 - - mkl=2023.1.0 - - mkl-service=2.4.0 - - mkl_fft=1.3.6 - - mkl_random=1.2.2 - - munkres=1.1.4 - - mysql=5.7.24 - - ncurses=6.4 - - nest-asyncio=1.5.6 - - nspr=4.35 - - nss=3.89.1 - - numpy=1.24.3 - - numpy-base=1.24.3 - - openssl=3.0.10 - - packaging=23.1 - - pandas=1.3.5 - - parso=0.8.3 - - pcre=8.45 - - pexpect=4.8.0 - - pickleshare=0.7.5 - - pillow=9.4.0 - - pip=23.2.1 - - platformdirs=3.10.0 - - pluggy=1.0.0 - - ply=3.11 - - prompt-toolkit=3.0.36 - - psutil=5.9.0 - - ptyprocess=0.7.0 - - pure_eval=0.2.2 - - py=1.11.0 - - pyerfa=2.0.0 - - pygments=2.15.1 - - pyparsing=3.0.9 - - pyqt=5.15.7 - - pyqt5-sip=12.11.0 - - pytest=7.1.2 - - pytest-runner=6.0.0 - - python=3.8.17 - - python-dateutil=2.8.2 - - python_abi=3.8 - - pytz=2022.7 - - pyyaml=6.0 - - pyzmq=25.1.0 - - qt-main=5.15.2 - - qt-webengine=5.15.9 - - qtwebkit=5.212 - - readline=8.2 - - regions=0.7 - - scipy=1.5.3 - - seaborn=0.12.2 - - setuptools=59.8.0 - - sip=6.6.2 - - six=1.16.0 - - sqlite=3.41.2 - - stack_data=0.2.0 - - tbb=2021.8.0 - - tk=8.6.12 - - toml=0.10.2 - - tomli=2.0.1 - - tornado=6.3.2 - - traitlets=5.7.1 - - typing_extensions=4.7.1 - - wcwidth=0.2.5 - - wheel=0.38.4 - - xz=5.4.2 - - yaml=0.2.5 - - zeromq=4.3.4 - - zipp=3.11.0 - - zlib=1.2.13 - - zstd=1.5.5 - + - python=3.8 + - ctools + - astropy + - scipy + - matplotlib + - pandas + - pyyaml + - pip: + - lxml -- GitLab From 5bbb50a5811dddbe561b3ef262821f2bee866fd0 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 14:37:52 +0200 Subject: [PATCH 32/98] update setup --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index d40ff4c..7bc2f83 100644 --- a/setup.py +++ b/setup.py @@ -8,11 +8,19 @@ from setuptools import setup, find_packages +entry_points = { + 'console_scripts': [ + 'base_simulator = astrort.simulator.base_simulator:main', + ] +} + setup( name='astrort', author='Ambra Di Piano ', package_dir={'astrort': 'astrort'}, + entry_points=entry_points, packages=find_packages(), include_package_data=True, license='BSD-3-Clause', + python_requires=">=3.8", ) \ No newline at end of file -- GitLab From 121a4ace3d410485a68cc6ff0d05b5d12aa2ce7a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 14:37:57 +0200 Subject: [PATCH 33/98] add pytest --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 5c197d4..b94cc61 100644 --- a/environment.yml +++ b/environment.yml @@ -7,5 +7,6 @@ dependencies: - matplotlib - pandas - pyyaml + - pytest - pip: - lxml -- GitLab From fe102ad33d80108d38bbbf96ec029aa86fb7c932 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 14:44:52 +0200 Subject: [PATCH 34/98] update rtasci --- rtasci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtasci b/rtasci index 20ce02e..e8d54ef 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 20ce02eb59e3598b72a14d85424a6e57ea3a3a55 +Subproject commit e8d54ef17a78e01f0881ef82b68ca771f07c4186 -- GitLab From c7ce99d6cb9ea9bef8ffa7e63852308f8b9afd46 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 14:46:58 +0200 Subject: [PATCH 35/98] del return --- astrort/testing/test_utils/test_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 4663962..1779e27 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -35,7 +35,6 @@ def test_get_instrument_fov(array): assert fov == 5 else: assert fov == 5 - return fov @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) def test_get_instrument_tev_range(array): @@ -48,4 +47,3 @@ def test_get_instrument_tev_range(array): assert erange == [5, 150] else: assert erange == [0.03, 150] - return erange \ No newline at end of file -- GitLab From 5178f374a65599fa1f6b2b583f96a766acc6189b Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 14:48:13 +0200 Subject: [PATCH 36/98] add slurm to configuration --- astrort/configure/check_configuration.py | 12 ++++++++++++ astrort/configure/test.yml | 12 +++++++++++- .../test_configure/test_check_configuration.py | 7 +++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index ae627f7..ebb9902 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -49,4 +49,16 @@ class CheckConfiguration(): assert self.conf['logging'].keys() == keys assert (type(self.conf['logging']['level']) == str or type(self.conf['logging']['level']) == int) assert type(self.conf['logging']['logfile']) == str + return self + + def check_slurm(self): + keys = ['nodes', 'tasks', 'cpus', 'mem', 'environment', 'name', 'account'] + assert self.conf['slurm'].keys() == keys + assert type(self.conf['slurm']['nodes']) == int + assert type(self.conf['slurm']['tasks']) == int + assert type(self.conf['slurm']['cpus']) == int + assert type(self.conf['slurm']['mem']) == str + assert type(self.conf['slurm']['environment']) == str + assert type(self.conf['slurm']['name']) == str + assert type(self.conf['slurm']['account']) == str return self \ No newline at end of file diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 3737d3e..e21d74f 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -5,7 +5,7 @@ simulator: prod: prod5-v0.1 pointing: random duration: 10 - samples: 5 + samples: 2 seed: 1 model: $TEMPLATES$/crab.xml output: /data01/homes/dipiano/astroRT/astrort/testing/tmp @@ -16,3 +16,13 @@ visibility: logging: level: CRITICAL logfile: + +slurm: + nodes: 5 + tasks: 1 + cpus: 2 + mem: 32G + environment: astrort + name: test + account: dipiano + diff --git a/astrort/testing/test_configure/test_check_configuration.py b/astrort/testing/test_configure/test_check_configuration.py index 15a787f..ea28a1b 100644 --- a/astrort/testing/test_configure/test_check_configuration.py +++ b/astrort/testing/test_configure/test_check_configuration.py @@ -42,6 +42,13 @@ class TestCheckConfiguration: type(e) == AssertionError def test_check_logging(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_logging() + except AssertionError as e: + type(e) == AssertionError + + def test_check_slurm(self, test_conf_file): configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_logging() -- GitLab From 931f3b8226176122f6a7f0ffcc4c913c1107f8fd Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:28:20 +0200 Subject: [PATCH 37/98] add partition --- astrort/configure/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index e21d74f..892ca33 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -25,4 +25,5 @@ slurm: environment: astrort name: test account: dipiano + partition: large -- GitLab From 59c9c89efc405a0977da3f7847c9ee923a7e3b72 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:28:28 +0200 Subject: [PATCH 38/98] add partition --- astrort/configure/check_configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index ebb9902..3a90e25 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -52,7 +52,7 @@ class CheckConfiguration(): return self def check_slurm(self): - keys = ['nodes', 'tasks', 'cpus', 'mem', 'environment', 'name', 'account'] + keys = ['nodes', 'tasks', 'cpus', 'mem', 'environment', 'name', 'account', 'partition'] assert self.conf['slurm'].keys() == keys assert type(self.conf['slurm']['nodes']) == int assert type(self.conf['slurm']['tasks']) == int @@ -61,4 +61,5 @@ class CheckConfiguration(): assert type(self.conf['slurm']['environment']) == str assert type(self.conf['slurm']['name']) == str assert type(self.conf['slurm']['account']) == str + assert type(self.conf['slurm']['partition']) == str return self \ No newline at end of file -- GitLab From 220491b8034b55937b563915893f26c584c02157 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:28:48 +0200 Subject: [PATCH 39/98] slurm jobs configuration utils and tests --- astrort/configure/slurmjobs.py | 43 ++++++++++++++ .../testing/test_configure/test_slurmjobs.py | 58 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 astrort/configure/slurmjobs.py create mode 100644 astrort/testing/test_configure/test_slurmjobs.py diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py new file mode 100644 index 0000000..db6faec --- /dev/null +++ b/astrort/configure/slurmjobs.py @@ -0,0 +1,43 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +from yaml import dump +from os import system +from os.path import join, dirname, abspath + +def make_configuration(jobname_conf, configuration, node_number): + configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + # write new configuration + with open(jobname_conf, 'w+') as f: + new_configuration = dump(configuration, f, default_flow_style=False) + +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): + # write sbatch + with open(jobname_sh, 'w+') as f: + f.write("#!/bin/bash") + #f.write("\n#SBATCH --nodes=1") + #f.write("\n#SBATCH --ntasks-per-node=1") + #f.write("\n#SBATCH --cpus-per-task=2") + f.write(f"\n#SBATCH --job-name={jobname}") + #f.write(f"\n#SBATCH --mem={slurmconf['memory']}") + f.write(f"\n#SBATCH --output={jobname_log}") + f.write(f"\n#SBATCH --account={slurmconf['account']}") + f.write(f"\n#SBATCH --partition={slurmconf['partition']}") + f.write(f"\n") + f.write(f"\nsource activate {slurmconf['environment']}") + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") + +def make_sbatch(jobname, configuration, node_number): + output = configuration['simulator']['output'] + jobname_sh = join(output, f"job_{jobname}.sh") + jobname_log = join(output, f"job_{jobname}.log") + jobname_conf = join(output, f"job_{jobname}.yml") + make_configuration(jobname_conf, configuration, node_number) + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) + system(f"sbatch {jobname_sh}") + \ No newline at end of file diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py new file mode 100644 index 0000000..36cd98a --- /dev/null +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -0,0 +1,58 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from shutil import rmtree +from os import listdir, makedirs +from os.path import isfile, join +from astrort.configure.slurmjobs import make_configuration, make_sh, make_sbatch +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.test_conf_file +def test_make_configuration(test_conf_file): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['simulator']['output']) + makedirs(conf['simulator']['output'], exist_ok=True) + + # make configurations + for node_number in range(conf['slurm']['nodes']): + node_number += 1 + jobname = f"{conf['slurm']['name']}_{node_number}" + jobname_conf = join(conf['simulator']['output'], f"job_{jobname}.yml") + make_configuration(jobname_conf, conf, node_number) + + # check output + expected_configurations = conf['slurm']['nodes'] + found_configurations = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.yml' in f and conf['slurm']['name'] in f]) + assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" + +@pytest.mark.test_conf_file +def test_make_sh(test_conf_file): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['simulator']['output']) + makedirs(conf['simulator']['output'], exist_ok=True) + + # make configurations + output = conf['simulator']['output'] + for node_number in range(conf['slurm']['nodes']): + node_number += 1 + jobname = f"{conf['slurm']['name']}_{node_number}" + jobname_sh = join(output, f"job_{jobname}.sh") + jobname_log = join(output, f"job_{jobname}.log") + jobname_conf = join(output, f"job_{jobname}.yml") + make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log) + + # check output + expected_sh = conf['slurm']['nodes'] + found_sh = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.sh' in f and conf['slurm']['name'] in f]) + assert found_sh == expected_sh, f"Expected {expected_sh} simulations, found {found_sh}" + -- GitLab From d53023f3e49015f71e61cc3a95ec73076427adec Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:29:04 +0200 Subject: [PATCH 40/98] slurm_submission option --- astrort/simulator/base_simulator.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 91f6eb7..93fec0b 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -11,6 +11,7 @@ from os import makedirs from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility from astrort.configure.logging import set_logger, get_log_level +from astrort.configure.slurmjobs import make_sbatch def base_simulator(configuration_file): configuration = load_yaml_conf(configuration_file) @@ -31,20 +32,30 @@ def base_simulator(configuration_file): log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") -def slurm_submission(configuration_file): +def slurm_submission(configuration_file, nodes): + configuration = load_yaml_conf(configuration_file) + log = set_logger(get_log_level(configuration['logging']['level'])) + # create output dir + log.info(f"Creating {configuration['simulator']['output']}") + makedirs(configuration['simulator']['output'], exist_ok=True) + # sbatch jobs per each nodes + configuration['slurm']['nodes'] = nodes + for node_number in range(configuration['slurm']['nodes']): + jobname = f"{configuration['slurm']['name']}_{node_number+1}" + make_sbatch(jobname, configuration, node_number+1) return +def main(configuration, nodes): + if nodes == 0: + base_simulator(configuration) + else: + slurm_submission(configuration, nodes) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='') parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) args = parser.parse_args() - - if args.nodes == 0: - base_simulator(args.configuration) - else: - slurm_submission(args.configuration) - - + main(args.configuration, args.nodes) -- GitLab From 99bdd18c34cb94e600b91ad28cec2f333bceb26f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:29:16 +0200 Subject: [PATCH 41/98] stop simulation utility --- astrort/simulator/stop_simulations.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 astrort/simulator/stop_simulations.py diff --git a/astrort/simulator/stop_simulations.py b/astrort/simulator/stop_simulations.py new file mode 100644 index 0000000..9c10a15 --- /dev/null +++ b/astrort/simulator/stop_simulations.py @@ -0,0 +1,17 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import sys +from os import system + +start_job_id = int(sys.argv[1]) +jobs = int(sys.argv[2]) + +for i in range(jobs): + system(f"scancel {start_job_id+i}") + -- GitLab From 5e9ef3f7149208660b71cbd6a23f927ca9924ce5 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 16:58:05 +0200 Subject: [PATCH 42/98] seed to string generalised --- astrort/configure/slurmjobs.py | 10 +++++----- astrort/testing/test_utils/test_utils.py | 2 +- astrort/utils/utils.py | 10 +++++----- astrort/utils/wrap.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index db6faec..a1814c8 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -11,10 +11,10 @@ from os import system from os.path import join, dirname, abspath def make_configuration(jobname_conf, configuration, node_number): - configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + configuration['simulator']['samples'] = node_number*configuration['simulator']['samples'] + 1 # write new configuration with open(jobname_conf, 'w+') as f: - new_configuration = dump(configuration, f, default_flow_style=False) + dump(configuration, f, default_flow_style=False) def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): # write sbatch @@ -34,9 +34,9 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): def make_sbatch(jobname, configuration, node_number): output = configuration['simulator']['output'] - jobname_sh = join(output, f"job_{jobname}.sh") - jobname_log = join(output, f"job_{jobname}.log") - jobname_conf = join(output, f"job_{jobname}.yml") + jobname_sh = join(output, f"{jobname}.sh") + jobname_log = join(output, f"{jobname}.slurm") + jobname_conf = join(output, f"{jobname}.yml") make_configuration(jobname_conf, configuration, node_number) make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 1779e27..c2a5b65 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -12,7 +12,7 @@ from astrort.utils.utils import * @pytest.mark.test_tmp_folder @pytest.mark.parametrize('samples', [3, 5, 8, 10]) def test_seeds_to_string_formatter(samples, test_tmp_folder): - name = seeds_to_string_formatter(samples, test_tmp_folder, name='test', seed=1) + name = seeds_to_string_formatter(samples, test_tmp_folder, name='test', seed=1, ext='fits') if samples <= 1e3: assert name == f"{test_tmp_folder}/test_001.fits" diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index c4a8fd5..23b9a8b 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -8,15 +8,15 @@ from os.path import join -def seeds_to_string_formatter(samples, output, name, seed): +def seeds_to_string_formatter(samples, output, name, seed, ext): if samples <= 1e3: - name = join(output, f"{name}_{seed:03d}.fits") + name = join(output, f"{name}_{seed:03d}.{ext}") elif samples <= 1e5: - name = join(output, f"{name}_{seed:05d}.fits") + name = join(output, f"{name}_{seed:05d}.{ext}") elif samples <= 1e8: - name = join(output, f"{name}_{seed:08d}.fits") + name = join(output, f"{name}_{seed:08d}.{ext}") else: - name = join(output, f"{name}_{seed}.fits") + name = join(output, f"{name}_{seed}.{ext}") return name def get_instrument_fov(instrument): diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 6354bfe..b087289 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -21,7 +21,7 @@ def configure_simulator_no_visibility(simulator, configuration): if '$TEMPLATES$' in configuration['model']: configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) simulator.model = configuration['model'] - simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed']) + simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], 'fits') simulator.caldb = configuration['prod'] simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) -- GitLab From 0a551118f72b5b30e7393a7346e2c6598f48ef5c Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 17:06:04 +0200 Subject: [PATCH 43/98] fix seed increment --- astrort/configure/slurmjobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index a1814c8..d90d06f 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -11,7 +11,7 @@ from os import system from os.path import join, dirname, abspath def make_configuration(jobname_conf, configuration, node_number): - configuration['simulator']['samples'] = node_number*configuration['simulator']['samples'] + 1 + configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -- GitLab From 49257e4300d622a5d4274fb878e0b316e63a7eb1 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 17:06:32 +0200 Subject: [PATCH 44/98] fix naming increase --- astrort/simulator/base_simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 93fec0b..ef2c04e 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -42,7 +42,7 @@ def slurm_submission(configuration_file, nodes): configuration['slurm']['nodes'] = nodes for node_number in range(configuration['slurm']['nodes']): jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_sbatch(jobname, configuration, node_number+1) + make_sbatch(jobname, configuration, node_number) return def main(configuration, nodes): -- GitLab From 4c94551071ee77f9170136b110522948b220407f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 18:28:13 +0200 Subject: [PATCH 45/98] min --- astrort/templates/crab.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/templates/crab.xml b/astrort/templates/crab.xml index 47843b8..bbeaed6 100644 --- a/astrort/templates/crab.xml +++ b/astrort/templates/crab.xml @@ -1,6 +1,6 @@ - + -- GitLab From 7e7d0ebb83ba5dd8399d0329f296932c5405ae54 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 18:28:40 +0200 Subject: [PATCH 46/98] update rtasci --- rtasci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtasci b/rtasci index e8d54ef..4d009f6 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit e8d54ef17a78e01f0881ef82b68ca771f07c4186 +Subproject commit 4d009f6979ada5eb2d69df50bcf98a0d27d42406 -- GitLab From def7528ca3323adb1c86f8ae745d7d213b198ae6 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 18:28:47 +0200 Subject: [PATCH 47/98] fix name --- astrort/configure/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 892ca33..d3e6015 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -1,5 +1,5 @@ simulator: - name: test + name: crab array: lst irf: North_z60_0.5h_LST prod: prod5-v0.1 -- GitLab From fc8645a24dd130235a5a36abfb2938eef7898e96 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 18:29:00 +0200 Subject: [PATCH 48/98] randomise_pointing --- astrort/testing/test_utils/test_wrap.py | 16 ++++++++++++++-- astrort/utils/wrap.py | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 54f28de..e3f82d2 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -7,9 +7,21 @@ # ***************************************************************************** import pytest -from astrort.utils.wrap import load_yaml_conf +import numpy as np +from astrort.utils.wrap import load_yaml_conf, randomise_pointing @pytest.mark.test_conf_file def test_load_yaml_conf(test_conf_file): configuration = load_yaml_conf(test_conf_file) - assert type(configuration) == dict \ No newline at end of file + assert type(configuration) == dict + +@pytest.mark.test_conf_file +def test_randomise_pointing(test_conf_file): + conf = load_yaml_conf(test_conf_file) + pointing = randomise_pointing(conf['simulator']) + assert type(pointing) == dict + for key in pointing.keys(): + assert key in ['ra', 'dec', 'offset'] + assert type(pointing['ra']) == type(np.float64(1)) + assert type(pointing['dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index b087289..c23f71d 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -7,9 +7,13 @@ # ***************************************************************************** import yaml +import numpy as np +import astropy.units as u from os.path import dirname, abspath, join, basename from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov from astrort.configure.check_configuration import CheckConfiguration +from rtasci.lib.RTAManageXml import ManageXml +from astropy.coordinates import SkyCoord def load_yaml_conf(yamlfile): with open(yamlfile) as f: @@ -28,3 +32,18 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.t = [0, configuration['duration']] simulator.seed = configuration['seed'] return simulator + +def randomise_pointing(simulator): + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + model_xml = ManageXml(xml=simulator['model']) + name = model_xml.getName() + source = model_xml.getRaDec() + del model_xml + # use astropy separation + ra, dec = source[0][0] * u.deg, source[1][0] * u.deg + source = SkyCoord(ra, dec, frame='icrs') + position_angle = 45 * u.deg + separation = np.random.random() * get_instrument_fov(simulator['array']) * u.deg + pointing = source.directional_offset_by(position_angle, separation) + return {'ra': pointing.ra.deg, 'dec': pointing.dec.deg, 'offset': separation.value} \ No newline at end of file -- GitLab From a37bbec93907809e03290455c8e2fee044a69980 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 20 Sep 2023 18:33:38 +0200 Subject: [PATCH 49/98] simulate random pointing --- astrort/utils/wrap.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index c23f71d..487558f 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -31,6 +31,13 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] simulator.seed = configuration['seed'] + if configuration['pointing'] == 'random': + point = randomise_pointing(configuration) + simulator.ra = point['ra'] + simulator.dec = point['dec'] + else: + simulator.ra = configuration['pointing']['ra'] + simulator.dec = configuration['pointing']['dec'] return simulator def randomise_pointing(simulator): -- GitLab From 377ac18e7791e78a486e4680fc6641255fd3d6d0 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:35:30 +0200 Subject: [PATCH 50/98] spacing --- astrort/testing/test_utils/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index c2a5b65..21fdcd9 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -39,6 +39,7 @@ def test_get_instrument_fov(array): @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) def test_get_instrument_tev_range(array): erange = get_instrument_tev_range(array) + if array == 'lst': assert erange == [0.03, 5] elif array == 'mst': -- GitLab From 882ca6eefbf8c52e72d819a637e4ac6c0a837c8e Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:35:42 +0200 Subject: [PATCH 51/98] add datfile --- astrort/configure/check_configuration.py | 3 ++- astrort/configure/test.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index 3a90e25..2e74c37 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -45,10 +45,11 @@ class CheckConfiguration(): return self def check_logging(self): - keys = ['level', 'logfile'] + keys = ['level', 'logfile', 'datfile'] assert self.conf['logging'].keys() == keys assert (type(self.conf['logging']['level']) == str or type(self.conf['logging']['level']) == int) assert type(self.conf['logging']['logfile']) == str + assert type(self.conf['logging']['datfile']) == str return self def check_slurm(self): diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index d3e6015..c8b7ba7 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -15,7 +15,8 @@ visibility: logging: level: CRITICAL - logfile: + logfile: test.log + datfile: test.dat slurm: nodes: 5 -- GitLab From 0ffb9ce0dbc963a4326afa0303c1076c184e9ee9 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:39:06 +0200 Subject: [PATCH 52/98] create logfile is missing --- astrort/configure/logging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py index 975f725..37df7fd 100644 --- a/astrort/configure/logging.py +++ b/astrort/configure/logging.py @@ -8,7 +8,7 @@ import logging from os import makedirs -from os.path import isdir, dirname +from os.path import isdir, dirname, isfile def set_logger(level, filename=None): log = logging.getLogger() @@ -22,6 +22,9 @@ def set_logger(level, filename=None): if filename is not None: if not isdir(dirname(filename)): makedirs(dirname(filename)) + if not isfile(filename): + f = open(filename, 'w+') + f.close() fileHandler = logging.FileHandler(filename) fileHandler.setFormatter(formatter) log.addHandler(fileHandler) -- GitLab From a4623e5616d2225e248df5a424445bcaf1c238b1 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:39:22 +0200 Subject: [PATCH 53/98] place log and dat in output --- astrort/configure/slurmjobs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index d90d06f..864199d 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -12,6 +12,8 @@ from os.path import join, dirname, abspath def make_configuration(jobname_conf, configuration, node_number): configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 + configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.log') + configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -- GitLab From b103328a4673ea647b2f355e0a06eef5764ad634 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:39:32 +0200 Subject: [PATCH 54/98] randomise pointing --- astrort/simulator/base_simulator.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index ef2c04e..8ae0579 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -8,14 +8,17 @@ import argparse from os import makedirs +from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, randomise_pointing_sim, get_point_source_info from astrort.configure.logging import set_logger, get_log_level from astrort.configure.slurmjobs import make_sbatch def base_simulator(configuration_file): configuration = load_yaml_conf(configuration_file) - log = set_logger(get_log_level(configuration['logging']['level'])) + logfile = join(configuration['simulator']['output'], configuration['logging']['logfile']) + datfile = configuration['logging']['logfile'].replace('.log', '.dat') + log = set_logger(get_log_level(configuration['logging']['level']), logfile) # create output dir log.info(f"Creating {configuration['simulator']['output']}") makedirs(configuration['simulator']['output'], exist_ok=True) @@ -23,10 +26,20 @@ def base_simulator(configuration_file): log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") for i in range(configuration['simulator']['samples']): simulator = RTACtoolsSimulation() + # check pointing option + if configuration['simulator']['pointing'] == 'random': + log.info(f"Randomising pointing coordinates") + point = randomise_pointing_sim(configuration['simulator']) + else: + log.info(f"Using fixed pointing coordinates") + point = get_point_source_info(configuration['simulator']) + configuration['simulator']['pointing'] = {'ra': point['point_ra'], 'dec': point['point_dec']} simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) simulator.run_simulation() log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete") configuration['simulator']['seed'] += 1 + # save simulation data + write_simulation_info(simulator, configuration, point, datfile) del simulator # end simulations log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") -- GitLab From 7811275ef9b9061dc247a26c9b5eaa95ea7d264d Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:40:06 +0200 Subject: [PATCH 55/98] randomise pointing and info source-pointing for safekeeping --- astrort/testing/test_utils/test_wrap.py | 30 +++++++++++++----- astrort/utils/wrap.py | 41 ++++++++++++++++++------- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index e3f82d2..dc361de 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -8,7 +8,7 @@ import pytest import numpy as np -from astrort.utils.wrap import load_yaml_conf, randomise_pointing +from astrort.utils.wrap import load_yaml_conf, randomise_pointing_sim, get_point_source_info @pytest.mark.test_conf_file def test_load_yaml_conf(test_conf_file): @@ -16,12 +16,28 @@ def test_load_yaml_conf(test_conf_file): assert type(configuration) == dict @pytest.mark.test_conf_file -def test_randomise_pointing(test_conf_file): +def test_randomise_pointing_sim(test_conf_file): conf = load_yaml_conf(test_conf_file) - pointing = randomise_pointing(conf['simulator']) + pointing = randomise_pointing_sim(conf['simulator']) assert type(pointing) == dict for key in pointing.keys(): - assert key in ['ra', 'dec', 'offset'] - assert type(pointing['ra']) == type(np.float64(1)) - assert type(pointing['dec']) == type(np.float64(1)) - assert type(pointing['offset']) == type(np.float64(1)) \ No newline at end of file + assert key in ['point_ra', 'point_dec', 'offset', 'source_ra', 'source_dec'] + assert type(pointing['point_ra']) == type(np.float64(1)) + assert type(pointing['point_dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) + assert type(pointing['source_ra']) == type(np.float64(1)) + assert type(pointing['source_dec']) == type(np.float64(1)) + +@pytest.mark.test_conf_file +def test_get_point_source_info(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + assert type(pointing) == dict + for key in pointing.keys(): + assert key in ['point_ra', 'point_dec', 'offset', 'source_ra', 'source_dec'] + assert type(pointing['point_ra']) == type(np.float64(1)) + assert type(pointing['point_dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) + assert type(pointing['source_ra']) == type(np.float64(1)) + assert type(pointing['source_dec']) == type(np.float64(1)) \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 487558f..51ec8b9 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -9,7 +9,7 @@ import yaml import numpy as np import astropy.units as u -from os.path import dirname, abspath, join, basename +from os.path import dirname, abspath, join, basename, isfile from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml @@ -31,20 +31,14 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] simulator.seed = configuration['seed'] - if configuration['pointing'] == 'random': - point = randomise_pointing(configuration) - simulator.ra = point['ra'] - simulator.dec = point['dec'] - else: - simulator.ra = configuration['pointing']['ra'] - simulator.dec = configuration['pointing']['dec'] + simulator.ra = configuration['pointing']['ra'] + simulator.dec = configuration['pointing']['dec'] return simulator -def randomise_pointing(simulator): +def randomise_pointing_sim(simulator): if '$TEMPLATES$' in simulator['model']: simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) model_xml = ManageXml(xml=simulator['model']) - name = model_xml.getName() source = model_xml.getRaDec() del model_xml # use astropy separation @@ -53,4 +47,29 @@ def randomise_pointing(simulator): position_angle = 45 * u.deg separation = np.random.random() * get_instrument_fov(simulator['array']) * u.deg pointing = source.directional_offset_by(position_angle, separation) - return {'ra': pointing.ra.deg, 'dec': pointing.dec.deg, 'offset': separation.value} \ No newline at end of file + return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + +def get_point_source_info(simulator): + if '$TEMPLATES$' in simulator['model']: + simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) + model_xml = ManageXml(xml=simulator['model']) + source = model_xml.getRaDec() + del model_xml + # use astropy separation + source = SkyCoord(source[0][0] * u.deg, source[1][0] * u.deg, frame='icrs') + pointing = SkyCoord(simulator['pointing']['ra'] * u.deg, simulator['pointing']['dec'] * u.deg, frame='icrs') + separation = source.separation(pointing) + return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + +def write_simulation_info(simulator, configuration, pointing, datfile): + name = configuration['simulator']['name'] + seed = simulator.seed + tstart, tstop = simulator.t + duration = configuration['simulator']['duration'] + point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] + if not isfile(datfile): + with open(datfile, 'w+') as f: + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset\n') + with open(datfile, 'a') as f: + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset}') + return \ No newline at end of file -- GitLab From e01ab485c3fe49ad74ec309a798cd3948e3e8556 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:41:19 +0200 Subject: [PATCH 56/98] fix datfile path --- astrort/simulator/base_simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 8ae0579..36d78d4 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -17,7 +17,7 @@ from astrort.configure.slurmjobs import make_sbatch def base_simulator(configuration_file): configuration = load_yaml_conf(configuration_file) logfile = join(configuration['simulator']['output'], configuration['logging']['logfile']) - datfile = configuration['logging']['logfile'].replace('.log', '.dat') + datfile = logfile.replace('.log', '.dat') log = set_logger(get_log_level(configuration['logging']['level']), logfile) # create output dir log.info(f"Creating {configuration['simulator']['output']}") -- GitLab From 21a339e4b5cb8a478b0725570157bed8cae133d8 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:43:33 +0200 Subject: [PATCH 57/98] don't save ctools logs --- astrort/utils/wrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 51ec8b9..722d082 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -33,6 +33,7 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.seed = configuration['seed'] simulator.ra = configuration['pointing']['ra'] simulator.dec = configuration['pointing']['dec'] + simulator.set_log = False return simulator def randomise_pointing_sim(simulator): -- GitLab From a79049026d4cad2b270cc8317a5068815de76194 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 12:44:38 +0200 Subject: [PATCH 58/98] fix lines in datfile --- astrort/utils/wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 722d082..fd1f080 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -72,5 +72,5 @@ def write_simulation_info(simulator, configuration, pointing, datfile): with open(datfile, 'w+') as f: f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset\n') with open(datfile, 'a') as f: - f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset}') + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset}\n') return \ No newline at end of file -- GitLab From d26c59b9d397ed0fabf29d1d2ac1f99322546c37 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 13:03:38 +0200 Subject: [PATCH 59/98] move pointing to its own function conf --- astrort/testing/test_utils/test_wrap.py | 4 +++- astrort/utils/wrap.py | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index dc361de..1eeb7a1 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -40,4 +40,6 @@ def test_get_point_source_info(test_conf_file): assert type(pointing['point_dec']) == type(np.float64(1)) assert type(pointing['offset']) == type(np.float64(1)) assert type(pointing['source_ra']) == type(np.float64(1)) - assert type(pointing['source_dec']) == type(np.float64(1)) \ No newline at end of file + assert type(pointing['source_dec']) == type(np.float64(1)) + + \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index fd1f080..c8761e4 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -31,11 +31,20 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] simulator.seed = configuration['seed'] - simulator.ra = configuration['pointing']['ra'] - simulator.dec = configuration['pointing']['dec'] simulator.set_log = False return simulator +def set_pointing(simulator, configuration, log): + if configuration['pointing'] == 'random': + log.info(f"Randomising pointing coordinates") + point = randomise_pointing_sim(configuration) + else: + log.info(f"Using fixed pointing coordinates") + point = get_point_source_info(configuration) + simulator.ra = point['point_ra'] + simulator.dec = point['point_dec'] + return simulator, point + def randomise_pointing_sim(simulator): if '$TEMPLATES$' in simulator['model']: simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) @@ -63,10 +72,10 @@ def get_point_source_info(simulator): return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} def write_simulation_info(simulator, configuration, pointing, datfile): - name = configuration['simulator']['name'] + name = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], '') seed = simulator.seed tstart, tstop = simulator.t - duration = configuration['simulator']['duration'] + duration = configuration['duration'] point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: -- GitLab From cfe8e1771b842acf534af23dbc427cf9c5885631 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 13:04:00 +0200 Subject: [PATCH 60/98] fix naming in datfile and more pointing to separate conf func --- astrort/simulator/base_simulator.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 36d78d4..2bccbd7 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -10,7 +10,7 @@ import argparse from os import makedirs from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation -from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, randomise_pointing_sim, get_point_source_info +from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing from astrort.configure.logging import set_logger, get_log_level from astrort.configure.slurmjobs import make_sbatch @@ -27,19 +27,14 @@ def base_simulator(configuration_file): for i in range(configuration['simulator']['samples']): simulator = RTACtoolsSimulation() # check pointing option - if configuration['simulator']['pointing'] == 'random': - log.info(f"Randomising pointing coordinates") - point = randomise_pointing_sim(configuration['simulator']) - else: - log.info(f"Using fixed pointing coordinates") - point = get_point_source_info(configuration['simulator']) - configuration['simulator']['pointing'] = {'ra': point['point_ra'], 'dec': point['point_dec']} + simulator, point = set_pointing(simulator, configuration['simulator'], log) + # complete configuration simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) simulator.run_simulation() log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete") configuration['simulator']['seed'] += 1 # save simulation data - write_simulation_info(simulator, configuration, point, datfile) + write_simulation_info(simulator, configuration['simulator'], point, datfile) del simulator # end simulations log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") -- GitLab From 48eb149028a9e7affbd512c6bf91a60cae6510bc Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 13:51:32 +0200 Subject: [PATCH 61/98] fix seed to format for name and files --- astrort/testing/test_utils/test_utils.py | 17 +++++++++++++++-- astrort/utils/utils.py | 13 ++++++++++++- astrort/utils/wrap.py | 6 +++--- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 21fdcd9..034b084 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -11,8 +11,8 @@ from astrort.utils.utils import * @pytest.mark.test_tmp_folder @pytest.mark.parametrize('samples', [3, 5, 8, 10]) -def test_seeds_to_string_formatter(samples, test_tmp_folder): - name = seeds_to_string_formatter(samples, test_tmp_folder, name='test', seed=1, ext='fits') +def test_seeds_to_string_formatter_files(samples, test_tmp_folder): + name = seeds_to_string_formatter_files(samples, test_tmp_folder, name='test', seed=1, ext='fits') if samples <= 1e3: assert name == f"{test_tmp_folder}/test_001.fits" @@ -23,6 +23,19 @@ def test_seeds_to_string_formatter(samples, test_tmp_folder): else: assert name == f"{test_tmp_folder}/test_1.fits" +@pytest.mark.parametrize('samples', [3, 5, 8, 10]) +def test_seeds_to_string_formatter(samples): + name = seeds_to_string_formatter(samples, name='test', seed=1) + + if samples <= 1e3: + assert name == f"test_001" + elif samples <= 1e5: + assert name == f"test_00001" + elif samples <= 1e8: + assert name == f"test_00000001" + else: + assert name == f"test_1" + @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'cta', 'north', 'south']) def test_get_instrument_fov(array): fov = get_instrument_fov(array) diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 23b9a8b..0cace74 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -8,7 +8,7 @@ from os.path import join -def seeds_to_string_formatter(samples, output, name, seed, ext): +def seeds_to_string_formatter_files(samples, output, name, seed, ext): if samples <= 1e3: name = join(output, f"{name}_{seed:03d}.{ext}") elif samples <= 1e5: @@ -19,6 +19,17 @@ def seeds_to_string_formatter(samples, output, name, seed, ext): name = join(output, f"{name}_{seed}.{ext}") return name +def seeds_to_string_formatter(samples, name, seed): + if samples <= 1e3: + name = join(f"{name}_{seed:03d}") + elif samples <= 1e5: + name = join(f"{name}_{seed:05d}") + elif samples <= 1e8: + name = join(f"{name}_{seed:08d}") + else: + name = join(f"{name}_{seed}") + return name + def get_instrument_fov(instrument): if instrument == 'lst': fov = 2.5 diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index c8761e4..3dd193d 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -10,7 +10,7 @@ import yaml import numpy as np import astropy.units as u from os.path import dirname, abspath, join, basename, isfile -from astrort.utils.utils import seeds_to_string_formatter, get_instrument_fov +from astrort.utils.utils import seeds_to_string_formatter_files, get_instrument_fov, seeds_to_string_formatter from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml from astropy.coordinates import SkyCoord @@ -25,7 +25,7 @@ def configure_simulator_no_visibility(simulator, configuration): if '$TEMPLATES$' in configuration['model']: configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) simulator.model = configuration['model'] - simulator.output = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], 'fits') + simulator.output = seeds_to_string_formatter_files(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], 'fits') simulator.caldb = configuration['prod'] simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) @@ -72,7 +72,7 @@ def get_point_source_info(simulator): return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} def write_simulation_info(simulator, configuration, pointing, datfile): - name = seeds_to_string_formatter(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], '') + name = seeds_to_string_formatter(configuration['samples'], configuration['name'], configuration['seed']) seed = simulator.seed tstart, tstop = simulator.t duration = configuration['duration'] -- GitLab From bb5e4e8edd43d3c40fe9afb05d2780fc3709b374 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 15:57:43 +0200 Subject: [PATCH 62/98] update rtasci --- rtasci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtasci b/rtasci index 4d009f6..2dee2a9 160000 --- a/rtasci +++ b/rtasci @@ -1 +1 @@ -Subproject commit 4d009f6979ada5eb2d69df50bcf98a0d27d42406 +Subproject commit 2dee2a9c5c364558196047ae857508bfcf6e40fe -- GitLab From c4db5ba6e4d0edd194f357d4aae382853febe854 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 15:57:56 +0200 Subject: [PATCH 63/98] fix energy range --- astrort/simulator/base_simulator.py | 14 +++++++++++--- astrort/utils/wrap.py | 10 ++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 2bccbd7..9377c06 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -7,6 +7,7 @@ # ***************************************************************************** import argparse +from time import time from os import makedirs from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation @@ -15,29 +16,36 @@ from astrort.configure.logging import set_logger, get_log_level from astrort.configure.slurmjobs import make_sbatch def base_simulator(configuration_file): + clock = time() configuration = load_yaml_conf(configuration_file) logfile = join(configuration['simulator']['output'], configuration['logging']['logfile']) datfile = logfile.replace('.log', '.dat') log = set_logger(get_log_level(configuration['logging']['level']), logfile) + log.info(f"Simulator configured, took {time() - clock} s") # create output dir - log.info(f"Creating {configuration['simulator']['output']}") + log.info(f"Output folder: {configuration['simulator']['output']}") makedirs(configuration['simulator']['output'], exist_ok=True) # start simulations log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") for i in range(configuration['simulator']['samples']): + clock_sim = time() simulator = RTACtoolsSimulation() # check pointing option simulator, point = set_pointing(simulator, configuration['simulator'], log) # complete configuration simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) + log.debug(f"IRF [{configuration['simulator']['irf']}] > energy range: {simulator.e} TeV") simulator.run_simulation() - log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete") + log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete, took {time() - clock_sim} s") configuration['simulator']['seed'] += 1 + # timing simulation + clock_sim = time() - clock_sim # save simulation data - write_simulation_info(simulator, configuration['simulator'], point, datfile) + write_simulation_info(simulator, configuration['simulator'], point, datfile, clock_sim) del simulator # end simulations log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") + log.info(f"Process complete, took {time() - clock} s") def slurm_submission(configuration_file, nodes): diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 3dd193d..fe5e8ba 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -10,9 +10,10 @@ import yaml import numpy as np import astropy.units as u from os.path import dirname, abspath, join, basename, isfile -from astrort.utils.utils import seeds_to_string_formatter_files, get_instrument_fov, seeds_to_string_formatter +from astrort.utils.utils import seeds_to_string_formatter_files, get_instrument_fov, seeds_to_string_formatter, get_instrument_tev_range from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml +from rtasci.lib.RTAUtils import check_energy_thresholds from astropy.coordinates import SkyCoord def load_yaml_conf(yamlfile): @@ -30,6 +31,7 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] + simulator.e = check_energy_thresholds(get_instrument_tev_range(configuration['array']), configuration['irf']) simulator.seed = configuration['seed'] simulator.set_log = False return simulator @@ -71,7 +73,7 @@ def get_point_source_info(simulator): separation = source.separation(pointing) return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} -def write_simulation_info(simulator, configuration, pointing, datfile): +def write_simulation_info(simulator, configuration, pointing, datfile, clock): name = seeds_to_string_formatter(configuration['samples'], configuration['name'], configuration['seed']) seed = simulator.seed tstart, tstop = simulator.t @@ -79,7 +81,7 @@ def write_simulation_info(simulator, configuration, pointing, datfile): point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: - f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset\n') + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset computation_time\n') with open(datfile, 'a') as f: - f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset}\n') + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {clock}\n') return \ No newline at end of file -- GitLab From fb1a62e3a1cebc890ad9c14c212a92859bd72e4f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 16:34:06 +0200 Subject: [PATCH 64/98] pick random irf --- astrort/testing/test_utils/test_utils.py | 6 ++++++ astrort/utils/utils.py | 10 +++++++++- astrort/utils/wrap.py | 12 ++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 034b084..529a20c 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -61,3 +61,9 @@ def test_get_instrument_tev_range(array): assert erange == [5, 150] else: assert erange == [0.03, 150] + +@pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'north', 'south']) +def test_select_irf(array): + irf = select_random_irf(array, 'prod5-v0.1') + assert array in irf.lower() + assert 'share/caldb/data/cta' in irf.lower() diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 0cace74..223e543 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -6,7 +6,9 @@ # Ambra Di Piano # ***************************************************************************** -from os.path import join +import random +from os import listdir +from os.path import join, expandvars def seeds_to_string_formatter_files(samples, output, name, seed, ext): if samples <= 1e3: @@ -51,3 +53,9 @@ def get_instrument_tev_range(array): else: erange = [0.03, 150] return erange + +def select_random_irf(array, prod): + path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') + irfs = listdir(path) + irf = random.choice([i for i in irfs if array in i.lower()]) + return join(path, irf) \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index fe5e8ba..f041e25 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -10,7 +10,7 @@ import yaml import numpy as np import astropy.units as u from os.path import dirname, abspath, join, basename, isfile -from astrort.utils.utils import seeds_to_string_formatter_files, get_instrument_fov, seeds_to_string_formatter, get_instrument_tev_range +from astrort.utils.utils import * from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml from rtasci.lib.RTAUtils import check_energy_thresholds @@ -28,7 +28,10 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.model = configuration['model'] simulator.output = seeds_to_string_formatter_files(configuration['samples'], configuration['output'], configuration['name'], configuration['seed'], 'fits') simulator.caldb = configuration['prod'] - simulator.irf = configuration['irf'] + if configuration['irf'] == 'random': + simulator.irf = select_random_irf(configuration['array'], configuration['prod']) + else: + simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] simulator.e = check_energy_thresholds(get_instrument_tev_range(configuration['array']), configuration['irf']) @@ -78,10 +81,11 @@ def write_simulation_info(simulator, configuration, pointing, datfile, clock): seed = simulator.seed tstart, tstop = simulator.t duration = configuration['duration'] + irf = configuration['irf'] point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: - f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset computation_time\n') + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf computation_time\n') with open(datfile, 'a') as f: - f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {clock}\n') + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {irf} {clock}\n') return \ No newline at end of file -- GitLab From eb54f85ce4fa013550efe423fc598d1b0153d0bd Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 17:05:32 +0200 Subject: [PATCH 65/98] fix irf adjustment --- astrort/simulator/base_simulator.py | 3 +-- astrort/testing/test_utils/test_utils.py | 8 ++++++-- astrort/utils/utils.py | 12 +++++++++++- astrort/utils/wrap.py | 9 +++++---- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 9377c06..c5a22e2 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -33,8 +33,7 @@ def base_simulator(configuration_file): # check pointing option simulator, point = set_pointing(simulator, configuration['simulator'], log) # complete configuration - simulator = configure_simulator_no_visibility(simulator, configuration['simulator']) - log.debug(f"IRF [{configuration['simulator']['irf']}] > energy range: {simulator.e} TeV") + simulator = configure_simulator_no_visibility(simulator, configuration['simulator'], log) simulator.run_simulation() log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete, took {time() - clock_sim} s") configuration['simulator']['seed'] += 1 diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 529a20c..4b8d3c5 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -7,6 +7,7 @@ # ***************************************************************************** import pytest +from os.path import isdir from astrort.utils.utils import * @pytest.mark.test_tmp_folder @@ -64,6 +65,9 @@ def test_get_instrument_tev_range(array): @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'north', 'south']) def test_select_irf(array): - irf = select_random_irf(array, 'prod5-v0.1') + prod = 'prod5-v0.1' + path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') + irf = select_random_irf(array, prod) assert array in irf.lower() - assert 'share/caldb/data/cta' in irf.lower() + assert 'share/caldb/data/cta' in join(path, irf).lower() + assert isdir(join(path, irf)) is True diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 223e543..95a2b66 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -54,8 +54,18 @@ def get_instrument_tev_range(array): erange = [0.03, 150] return erange +def adjust_tev_range_to_irf(erange, irf): + # minimum energy + if "z60" in irf and erange[0] < 0.11: + erange[0] = 0.11 + elif "z40" in irf and erange[0] < 0.04: + erange[0] = 0.04 + elif "z20" in irf and erange[0] < 0.03: + erange[0] = 0.03 + return erange + def select_random_irf(array, prod): path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') irfs = listdir(path) irf = random.choice([i for i in irfs if array in i.lower()]) - return join(path, irf) \ No newline at end of file + return irf \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index f041e25..0ce8da3 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -22,7 +22,7 @@ def load_yaml_conf(yamlfile): CheckConfiguration(configuration=configuration) return configuration -def configure_simulator_no_visibility(simulator, configuration): +def configure_simulator_no_visibility(simulator, configuration, log): if '$TEMPLATES$' in configuration['model']: configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) simulator.model = configuration['model'] @@ -30,21 +30,22 @@ def configure_simulator_no_visibility(simulator, configuration): simulator.caldb = configuration['prod'] if configuration['irf'] == 'random': simulator.irf = select_random_irf(configuration['array'], configuration['prod']) + log.info(f"Randomising instrument response function [{simulator.irf}]") else: simulator.irf = configuration['irf'] simulator.fov = get_instrument_fov(configuration['array']) simulator.t = [0, configuration['duration']] - simulator.e = check_energy_thresholds(get_instrument_tev_range(configuration['array']), configuration['irf']) + simulator.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['array']), simulator.irf) + log.info(f"Verified energy range {simulator.e}") simulator.seed = configuration['seed'] simulator.set_log = False return simulator def set_pointing(simulator, configuration, log): if configuration['pointing'] == 'random': - log.info(f"Randomising pointing coordinates") point = randomise_pointing_sim(configuration) + log.info(f"Randomising pointing coordinates [{point['point_ra']}, {point['point_dec']}]") else: - log.info(f"Using fixed pointing coordinates") point = get_point_source_info(configuration) simulator.ra = point['point_ra'] simulator.dec = point['point_dec'] -- GitLab From 7614185abc2861fb1d2e9cd54590a57860951604 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 21 Sep 2023 17:17:00 +0200 Subject: [PATCH 66/98] fix test adjust tev range --- astrort/testing/test_utils/test_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 4b8d3c5..9c6d324 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -63,11 +63,23 @@ def test_get_instrument_tev_range(array): else: assert erange == [0.03, 150] +@pytest.mark.parametrize('zenith', ['irf_z60', 'irf_z40', 'irf_z20']) +def test_adjust_tev_range_to_irf(zenith): + erange = adjust_tev_range_to_irf([0.03, 150], zenith) + + if 'z60' in zenith: + assert erange[0] == 0.11 + elif 'z60' in zenith: + assert erange[0] == 0.04 + elif 'z20' in zenith: + assert erange[0] == 0.03 + @pytest.mark.parametrize('array', ['lst', 'mst', 'sst', 'north', 'south']) def test_select_irf(array): prod = 'prod5-v0.1' path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') irf = select_random_irf(array, prod) + assert array in irf.lower() assert 'share/caldb/data/cta' in join(path, irf).lower() assert isdir(join(path, irf)) is True -- GitLab From 0b2b272ff3b5097ed7d5ec3d3481fe7c4fb338fe Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 13:42:04 +0200 Subject: [PATCH 67/98] rename with simulator descriptor --- astrort/configure/slurmjobs.py | 8 +++++--- astrort/simulator/base_simulator.py | 4 ++-- astrort/testing/test_configure/test_slurmjobs.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 864199d..f5c58a2 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -11,14 +11,16 @@ from os import system from os.path import join, dirname, abspath def make_configuration(jobname_conf, configuration, node_number): + # simulator configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 + # logging configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.log') configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): +def make_simulator_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): # write sbatch with open(jobname_sh, 'w+') as f: f.write("#!/bin/bash") @@ -34,12 +36,12 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): f.write(f"\nsource activate {slurmconf['environment']}") f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") -def make_sbatch(jobname, configuration, node_number): +def make_simulator_sbatch(jobname, configuration, node_number): output = configuration['simulator']['output'] jobname_sh = join(output, f"{jobname}.sh") jobname_log = join(output, f"{jobname}.slurm") jobname_conf = join(output, f"{jobname}.yml") make_configuration(jobname_conf, configuration, node_number) - make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) + make_simulator_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") \ No newline at end of file diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index c5a22e2..b41c240 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -13,7 +13,7 @@ from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing from astrort.configure.logging import set_logger, get_log_level -from astrort.configure.slurmjobs import make_sbatch +from astrort.configure.slurmjobs import make_simulator_sbatch def base_simulator(configuration_file): clock = time() @@ -57,7 +57,7 @@ def slurm_submission(configuration_file, nodes): configuration['slurm']['nodes'] = nodes for node_number in range(configuration['slurm']['nodes']): jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_sbatch(jobname, configuration, node_number) + make_simulator_sbatch(jobname, configuration, node_number) return def main(configuration, nodes): diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index 36cd98a..c580685 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -10,7 +10,7 @@ import pytest from shutil import rmtree from os import listdir, makedirs from os.path import isfile, join -from astrort.configure.slurmjobs import make_configuration, make_sh, make_sbatch +from astrort.configure.slurmjobs import make_configuration, make_simulator_sh, make_simulator_sbatch from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file @@ -34,7 +34,7 @@ def test_make_configuration(test_conf_file): assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" @pytest.mark.test_conf_file -def test_make_sh(test_conf_file): +def test_make_simulator_sh(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) @@ -49,7 +49,7 @@ def test_make_sh(test_conf_file): jobname_sh = join(output, f"job_{jobname}.sh") jobname_log = join(output, f"job_{jobname}.log") jobname_conf = join(output, f"job_{jobname}.yml") - make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log) + make_simulator_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log) # check output expected_sh = conf['slurm']['nodes'] -- GitLab From c9766620cdfd1f4f171751b075d803eac4be98ed Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 13:58:41 +0200 Subject: [PATCH 68/98] add mapper configuration --- astrort/configure/check_configuration.py | 13 ++++++++++++- astrort/configure/test.yml | 7 +++++++ .../test_configure/test_check_configuration.py | 7 +++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index 2e74c37..f6b721b 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -25,7 +25,7 @@ class CheckConfiguration(): return self def check_simulator(self): - keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model'] + keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model', 'output'] assert self.conf['simulator'].keys() == keys assert type(self.conf['simulator']['name']) == str assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south'] @@ -36,6 +36,7 @@ class CheckConfiguration(): assert type(self.conf['simulator']['samples']) == int assert type(self.conf['simulator']['seed']) == int assert type(self.conf['simulator']['model']) == str + assert type(self.conf['simulator']['output']) == str return self def check_visibility(self): @@ -63,4 +64,14 @@ class CheckConfiguration(): assert type(self.conf['slurm']['name']) == str assert type(self.conf['slurm']['account']) == str assert type(self.conf['slurm']['partition']) == str + return self + + def check_mapper(self): + keys = ['exposure', 'smooth', 'center', 'region', 'output'] + assert self.conf['mapper'].keys() == keys + assert type(self.conf['mapper']['exposure']) == int + assert type(self.conf['mapper']['smooth']) == (float or int) + assert type(self.conf['mapper']['center']) in ['pointing', 'source'] + assert type(self.conf['mapper']['region']) == bool + assert type(self.conf['mapper']['output']) == str return self \ No newline at end of file diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index c8b7ba7..a13f31f 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -10,6 +10,13 @@ simulator: model: $TEMPLATES$/crab.xml output: /data01/homes/dipiano/astroRT/astrort/testing/tmp +mapper: + exposure: 10 + smooth: 1 + center: pointing + region: false + output: /data01/homes/dipiano/astroRT/astrort/testing/tmp + visibility: start_time: '2030-01-01T00:00:00' diff --git a/astrort/testing/test_configure/test_check_configuration.py b/astrort/testing/test_configure/test_check_configuration.py index ea28a1b..f31839b 100644 --- a/astrort/testing/test_configure/test_check_configuration.py +++ b/astrort/testing/test_configure/test_check_configuration.py @@ -52,5 +52,12 @@ class TestCheckConfiguration: configuration = load_yaml_conf(test_conf_file) try: CheckConfiguration(configuration=configuration).check_logging() + except AssertionError as e: + type(e) == AssertionError + + def test_check_mapper(self, test_conf_file): + configuration = load_yaml_conf(test_conf_file) + try: + CheckConfiguration(configuration=configuration).check_mapper() except AssertionError as e: type(e) == AssertionError \ No newline at end of file -- GitLab From c23c555d41188cc1383847a7ffbef0ff8ffb58cb Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 13:59:18 +0200 Subject: [PATCH 69/98] plotter class --- astrort/utils/plotting.py | 245 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 astrort/utils/plotting.py diff --git a/astrort/utils/plotting.py b/astrort/utils/plotting.py new file mode 100644 index 0000000..9472805 --- /dev/null +++ b/astrort/utils/plotting.py @@ -0,0 +1,245 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import numpy as np +import matplotlib as mpl +mpl.use('Agg') +import matplotlib.pyplot as plt +from astropy.io import fits +from scipy.ndimage import gaussian_filter +from astropy.wcs import WCS +from matplotlib.colors import SymLogNorm +from astropy import units as u + +class Plotter(): + def __init__(self, logger) -> None: + self.set_logger(logger=logger) + pass + + def set_logger(self, logger): + self.log = logger + return self + + def set_target(self, ra=83.63, dec=22.01, rad=0.2, color='white'): + self.target = {'ra': ra, 'dec': dec, 'rad': rad, 'color': color} + return self + + def set_target_from_dict(self, target, rad=0.2, color='white'): + self.target = target + self.target['rad'] = rad + self.target['color'] = color + return self + + def set_pointing(self, ra=83.63, dec=22.51): + self.pointing = {'ra': ra, 'dec': dec, 'ftm': 'k+'} + return self + + def set_pointing_from_dict(self, pointing): + self.pointing = pointing + return self + + def __check_target(self): + assert isinstance(self.target, dict) is True, 'check that you have set the target correctly.' + return self + + def __check_pointing(self): + assert isinstance(self.pointing, dict) is True, 'check that you have set the pointing correctly.' + return self + + def set_path(self, path): + self.path = path + return self + + def load_fits(self, filename, extension='SKYMAP'): + h = fits.open(filename) + h.info() + data = h[extension].data + return data + + def heatmap_with_smoothing(self, x, y, sigma, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + heatmap = gaussian_filter(heatmap, sigma=sigma) + return heatmap.T, extent + + def heatmap(self, x, y, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + return heatmap.T, extent + + def get_extent(self, roi): + extent = [self.pointing['ra']-roi, self.pointing['ra']+roi, self.pointing['dec']-roi, self.pointing['dec']+roi] + return extent + + def set_wcs(self, point_ref, pixelsize): + w = WCS(naxis=2) + w.wcs.ctype = ['RA---CAR', 'DEC--CAR'] + w.wcs.crpix = [point_ref, point_ref] + w.wcs.crval = [self.pointing['ra'], self.pointing['dec']] + w.wcs.cdelt = [-pixelsize, pixelsize] + w.wcs.lonpole = 0.0 + w.wcs.latpole = 67.49 + return w + + def reshape_coords(self, ra, dec): + assert np.shape(ra) == np.shape(dec), 'RA and DEC shapes do not match' + coords = np.array([ra, dec]) + coords = coords.reshape((len(ra), 2)) + return coords + + def reshape_ra_dec(self, coords): + ra, dec = [], [] + for i in range(len(coords)): + ra.append(coords[i][0]) + dec.append(coords[i][1]) + ra = np.array(ra) + dec = np.array(dec) + return ra, dec + + def counts_map(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): + self.__check_pointing() + extent = self.get_extent(roi) + # counts map + data = self.get_phlist_data(file=file) + # select counts map + data = self.phlist_selection_cuts(data=data, trange=trange, erange=erange, roi=roi) + # axis and binning + ra = np.array(data.field('RA')).flatten() + dec = np.array(data.field('DEC')).flatten() + bins = int(roi*2/pixelsize) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot() + if sigma != 0: + hist = self.heatmap_with_smoothing(ra, dec, sigma=sigma, extent=extent, bins=bins) + else: + hist = self.heatmap(ra, dec, extent=extent, bins=bins) + img = ax.imshow(hist[0], norm=SymLogNorm(1, base=10), origin='lower', extent=extent, cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + # pointing + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + ax.set_xlim((extent[0], extent[1])) + ax.set_ylim((extent[2], extent[3])) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def counts_map_with_wcs(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): + self.__check_pointing() + extent = self.get_extent(roi=roi) + # counts map + data = self.get_phlist_data(file=file) + # select counts map + data = self.phlist_selection_cuts(data=data, trange=trange, erange=erange, roi=roi) + # axis and binning + ra = np.array(data.field('RA')).flatten() + dec = np.array(data.field('DEC')).flatten() + bins = int(roi*2/pixelsize) + # wcs + wcs = self.set_wcs(point_ref=bins/2+0.5, pixelsize=pixelsize) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot(projection=wcs) + if sigma != 0: + hist = self.heatmap_with_smoothing(ra, dec, sigma=sigma, extent=extent, bins=bins) + else: + hist = self.heatmap(ra, dec, extent=extent, bins=bins) + ax.coords[0].set_format_unit(u.deg) + ax.coords[1].set_format_unit(u.deg) + img = ax.imshow(hist[0], norm=SymLogNorm(1, base=10), interpolation='gaussian', extent=extent, cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + # pointing + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def plot_fits_skymap(self, file, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', logbar=False): + + # get map + data, wcs = self.get_skymap_data_with_wcs(file=file) + # plot + fig = plt.figure(figsize=figsize) + ax = plt.subplot(projection=wcs) + ax.coords[0].set_format_unit(u.deg) + ax.coords[1].set_format_unit(u.deg) + if logbar: + img = plt.imshow(data, norm=SymLogNorm(1, base=10), origin='lower', interpolation='gaussian', cmap=cmap) + else: + img = plt.imshow(data, interpolation='gaussian', cmap=cmap) + cb = plt.colorbar(img, ax=ax) + # axis + ax.tick_params(axis='both', labelsize=fontsize) + cb.ax.tick_params(labelsize=fontsize) + cb.set_label('counts', fontsize=fontsize) + ax.set_xlabel(xlabel, fontsize=fontsize) + ax.set_ylabel(ylabel, fontsize=fontsize) + ax.set_title(title, fontsize=fontsize) + ax.set_aspect('equal') + #ax.invert_xaxis() + ax.grid(color='grey', ls='solid') + #fig.tight_layout() + fig.savefig(name) + plt.close() + return self + + def get_skymap_data_with_wcs(self, file): + with fits.open(file) as h: + try: + w = WCS(h['SKYMAP'].header) + print(h['SKYMAP'].header) + data = h['SKYMAP'].data + except KeyError as e: + self.log.error(f'Missing "SKYMAP" extention, the input file may not be a compatible counts map. {e}') + raise KeyError(f'Missing "SKYMAP" extention, the input file may not be a compatible counts map. {e}') + return data, w + + def get_phlist_data(self, file): + with fits.open(file) as h: + try: + data = h['EVENTS'].data + except KeyError as e: + self.log.error(f'Missin "EVENTS" extention, the input file may not be a compatible photon list. {e}') + raise KeyError(f'Missin "EVENTS" extention, the input file may not be a compatible photon list. {e}') + if len(data) == 0: + self.log.warning("Empty photon list.") + return data + + def phlist_selection_cuts(self, data, trange=None, erange=None, roi=None): + if trange != None: + data = data[(data['TIME'] > trange[0]) & (data['TIME'] < trange[1])] + if erange != None: + data = data[(data['ENERGY'] > erange[0]) & (data['ENERGY'] < erange[1])] + if roi != None: + data = data[(np.abs(np.abs(data['RA'])-np.abs(self.pointing['ra'])) < roi)] + data = data[(np.abs(np.abs(data['DEC'])-np.abs(self.pointing['dec'])) < roi)] + if len(data) == 0: + self.log.warning("Empty photon list selection.") + return data -- GitLab From fec1784d37dd606091f103e8d566f9074178f3c4 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 13:59:26 +0200 Subject: [PATCH 70/98] mapper class --- astrort/utils/mapping.py | 226 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 astrort/utils/mapping.py diff --git a/astrort/utils/mapping.py b/astrort/utils/mapping.py new file mode 100644 index 0000000..4ce7c15 --- /dev/null +++ b/astrort/utils/mapping.py @@ -0,0 +1,226 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import shutil +import numpy as np +from astropy.io import fits +from scipy.ndimage import gaussian_filter + +class Mapper(): + '''Class that read, writes and manages the FITS data format.''' + def __init__(self, logger) -> None: + self.hdr_fits_keys = ['NAXIS1', 'NAXIS2', 'CRPIX1', 'CRPIX2', 'CDELT1', 'CDELT2', 'CRVAL1', 'CRVAL2', 'DATE-OBS', 'TIME-OBS', 'DATE-END', 'TIME-END', 'TELAPSE', 'ONTIME', 'LIVETIME', 'EXPOSURE', 'DEADC', 'E_MIN', 'E_MAX'] + + self.set_logger(logger=logger) + pass + + def set_logger(self, logger): + self.log = logger + return self + + def empty_header(self, hdr): + hdr['NAXIS1'] = 0 + hdr['NAXIS2'] = 0 + hdr['CRPIX1'] = 0.0 + hdr['CRPIX2'] = 0.0 + hdr['CDELT1'] = 0.0 + hdr['CDELT2'] = 0.0 + hdr['CRVAL1'] = 0.0 + hdr['CRVAL2'] = 0.0 + #hdr['LONPOLE'] = 0.0 + #hdr['LATPOLE'] = 0.0 + hdr['CREATOR'] = 'RTAPH' + hdr['DATE-OBS'] = '2000-01-01' + hdr['TIME-OBS'] = '00:00:00' + hdr['DATE-END'] = '2000-01-01' + hdr['TIME-END'] = '00:00:00' + hdr['TELAPSE'] = 0.0 + hdr['ONTIME'] = 0.0 + hdr['LIVETIME'] = 0.0 + hdr['EXPOSURE'] = 0.0 + hdr['DEADC'] = 0.0 + hdr['E_MIN'] = 0.0 + hdr['E_MAX'] = 0.0 + return hdr + + def get_template(self): + '''TODO''' + return + + def convert_countmap_in_template(self, skymap, template): + shutil.copy(skymap, template) + with fits.open(template, mode='update') as h: + h['SKYMAP'].data = np.empty_like(h['SKYMAP'].data) + h['SKYMAP'].header = self.empty_header(h['SKYMAP'].header) + h.flush() + return self + + def update_countmap_hdr(self, hdr, hdr_info): + hdr['NAXIS1'] = hdr_info['nbins'] + hdr['NAXIS2'] = hdr_info['nbins'] + hdr['CRPIX1'] = hdr_info['xref'] + hdr['CRPIX2'] = hdr_info['yref'] + hdr['CDELT1'] = hdr_info['pixelsize'] + hdr['CDELT2'] = hdr_info['pixelsize'] + hdr['CRVAL1'] = hdr_info['point_ra'] + hdr['CRVAL2'] = hdr_info['point_dec'] + #hdr['LONPOLE'] = hdr_info['lonpole'] + #hdr['LATPOLE'] = hdr_info['latpole'] + hdr['CREATOR'] = 'RTAPH' + hdr['DATE-OBS'] = hdr_info['date_obs'] + hdr['TIME-OBS'] = hdr_info['time_obs'] + hdr['DATE-END'] = hdr_info['date_end'] + hdr['TIME-END'] = hdr_info['time_end'] + hdr['TELAPSE'] = hdr_info['telapse'] + hdr['ONTIME'] = hdr_info['ontime'] + hdr['LIVETIME'] = hdr_info['livetime'] + hdr['EXPOSURE'] = hdr_info['exposure'] + hdr['DEADC'] = hdr_info['deadc'] + hdr['E_MIN'] = hdr_info['emin'] + hdr['E_MAX'] = hdr_info['emax'] + return hdr + + def get_header_info_from_dl3(self, hdr): + hdr_info = {} + for k in self.hdr_fits_keys: + hdr_info[k] = hdr[k] + return hdr_info + + def get_dl3_hdr(self, dl3_file): + with fits.open(dl3_file) as h: + hdr = h['EVENTS'].header + return hdr + + def set_dl3_hdr(self, dl3_file, hdr_fits): + with fits.open(dl3_file, mode='update') as h: + for k in hdr_fits.keys(): + h['EVENTS'].header[k] = hdr_fits[k] + h.flush() + return self + + def get_dl3_data(self, dl3_file): + with fits.open(dl3_file) as h: + data = h['EVENTS'].data + return data + + def set_dl3_data(self, dl3_file, data, GTI=None): + with fits.open(dl3_file, mode='update') as h: + h['EVENTS'].data = data + if GTI is not None: + h['GTI'].data[0] = GTI + h.flush() + return self + + def get_dl4_hdr(self, dl4_file): + with fits.open(dl4_file) as h: + hdr = h['SKYMAP'].header + return hdr + + def set_dl4_hdr(self, dl4_file, hdr_fits): + with fits.open(dl4_file, mode='update') as h: + for k in hdr_fits.keys(): + h['SKYMAP'].header[k] = hdr_fits[k] + h.flush() + return self + + def get_dl4_data(self, dl4_file): + with fits.open(dl4_file) as h: + data = h['SKYMAP'].data + return data + + def set_dl4_data(self, dl4_file, data): + with fits.open(dl4_file, mode='update') as h: + h['SKYMAP'].data = data + h.flush() + return self + + def get_binning_size(self, maproi=5, pixelsize=0.02): + nbins = int(maproi*2/pixelsize) + return nbins + + def get_pixel_reference_point(self, nbins): + return nbins/2+0.5, nbins/2+0.5 + + def store_hdr_in_dict(self, dl3_hdr, maproi=5, pixelsize=0.02): + hdr_fits = {} + nbins = self.get_binning_size(maproi=maproi, pixelsize=pixelsize) + hdr_fits['NAXIS1'] = nbins + hdr_fits['NAXIS2'] = nbins + xref, yref = self.get_pixel_reference_point(nbins=nbins) + hdr_fits['CRPIX1'] = xref + hdr_fits['CRPIX2'] = yref + hdr_fits['CDELT1'] = pixelsize + hdr_fits['CDELT2'] = pixelsize + hdr_fits['CRVAL1'] = float(dl3_hdr['RA_PNT']) + hdr_fits['CRVAL2'] = float(dl3_hdr['DEC_PNT']) + hdr_fits['CREATOR'] = 'ASTRORT' + hdr_fits['TSTART'] = float(dl3_hdr['TSTART']) + hdr_fits['TSTOP'] = float(dl3_hdr['TSTOP']) + hdr_fits['TELAPSE'] = dl3_hdr['TELAPSE'] + hdr_fits['ONTIME'] = dl3_hdr['ONTIME'] + hdr_fits['LIVETIME'] = dl3_hdr['LIVETIME'] + hdr_fits['EXPOSURE'] = dl3_hdr['LIVETIME'] + hdr_fits['DEADC'] = dl3_hdr['DEADC'] + erange = dl3_hdr['DSVAL2'].split(':') + hdr_fits['E_MIN'] = float(erange[0]) + hdr_fits['E_MAX'] = float(erange[1]) + hdr_fits['RADESYS'] = dl3_hdr['RADECSYS'] + return hdr_fits + + def get_countmap_in_fits(self, dl3_file, template, pixelsize=0.02, maproi=5, trange=None, erange=None, sigma=1, fitsname='skymap.fits'): + shutil.copy(template, fitsname) + dl3_hdr = self.get_dl3_hdr(dl3_file=dl3_file) + pointing = {'ra': float(dl3_hdr['RA_PNT']), 'dec': float(dl3_hdr['DEC_PNT'])} + hdr_fits = self.store_hdr_in_dict(dl3_hdr=dl3_hdr, maproi=maproi, pixelsize=pixelsize) + dl3_data = self.get_dl3_data(dl3_file=dl3_file) + dl3_data = self.selection_cuts(dl3_data=dl3_data, pointing=pointing, trange=trange, erange=erange, maproi=maproi) + if sigma != 0: + dl4_data = self.from_dl3_to_dl4(dl3_data=dl3_data, pointing=pointing, maproi=maproi, pixelsize=0.02, sigma=sigma) + else: + dl4_data = self.from_dl3_to_dl4(dl3_data=dl3_data, pointing=pointing, maproi=maproi, pixelsize=pixelsize) + self.set_dl4_hdr(dl4_file=fitsname, hdr_fits=hdr_fits) + self.set_dl4_data(dl4_file=fitsname, data=dl4_data) + return + + def get_extent(self, pointing, roi): + extent = [pointing['ra']-roi, pointing['ra']+roi, pointing['dec']-roi, pointing['dec']+roi] + return extent + + def heatmap_with_smoothing(self, x, y, extent, sigma=1, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + heatmap = gaussian_filter(heatmap, sigma=sigma) + return heatmap.T + + def heatmap(self, x, y, extent, bins=1000): + r = [[extent[0], extent[1]], [extent[2], extent[3]]] + heatmap, xedges, yedges = np.histogram2d(x, y, bins=bins) + return heatmap.T + + def from_dl3_to_dl4(self, dl3_data, pointing, maproi=5, pixelsize=0.02, sigma=1): + ra = np.array(dl3_data.field('RA')).flatten() + dec = np.array(dl3_data.field('DEC')).flatten() + nbins = self.get_binning_size(maproi=maproi, pixelsize=pixelsize) + extent = self.get_extent(pointing=pointing, roi=maproi) + if sigma != 0: + dl4_data = self.heatmap_with_smoothing(ra, dec, extent=extent, bins=nbins, sigma=sigma) + else: + dl4_data = self.heatmap(ra, dec, extent=extent, bins=nbins) + return dl4_data + + def selection_cuts(self, dl3_data, pointing, trange=None, erange=None, maproi=None): + if trange != None: + dl3_data = dl3_data[(dl3_data['TIME'] > trange[0]) & (dl3_data['TIME'] < trange[1])] + if erange != None: + dl3_data = dl3_data[(dl3_data['ENERGY'] > erange[0]) & (dl3_data['ENERGY'] < erange[1])] + if maproi != None: + dl3_data = dl3_data[(np.abs(np.abs(dl3_data['RA'])-np.abs(pointing['ra'])) < maproi)] + dl3_data = dl3_data[(np.abs(np.abs(dl3_data['DEC'])-np.abs(pointing['dec'])) < maproi)] + if len(dl3_data) == 0: + self.log.warning("Empty photon list selection.") + return dl3_data -- GitLab From 6db35cab1896481da9d6441cde6f92203cee8865 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 14:47:25 +0200 Subject: [PATCH 71/98] merge datfile and test --- astrort/testing/test_utils/test_wrap.py | 40 ++++++++++++++++++++++++- astrort/utils/wrap.py | 25 +++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 1eeb7a1..2df9d6e 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -7,8 +7,11 @@ # ***************************************************************************** import pytest +import logging import numpy as np -from astrort.utils.wrap import load_yaml_conf, randomise_pointing_sim, get_point_source_info +from astrort.utils.wrap import * +from astrort.configure.logging import set_logger +from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation @pytest.mark.test_conf_file def test_load_yaml_conf(test_conf_file): @@ -42,4 +45,39 @@ def test_get_point_source_info(test_conf_file): assert type(pointing['source_ra']) == type(np.float64(1)) assert type(pointing['source_dec']) == type(np.float64(1)) +@pytest.mark.test_conf_file +def test_write_simulation_info(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + datfile = join(conf['simulator']['output'], 'tmp.dat') + sim = RTACtoolsSimulation() + clock = 1 + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + del sim + +@pytest.mark.test_tmp_folder +@pytest.mark.test_conf_file +def test_merge_simulation_info(test_conf_file, test_tmp_folder): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + sim = RTACtoolsSimulation() + clock = 1 + for i in range(5): + sim.seed = i + datfile = join(conf['simulator']['output'], f'job_{i}.dat') + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) + merge_simulation_info(conf['simulator'], log) + assert isfile(join(conf['simulator']['output'], 'merged_sim_data.dat')) + del sim + +def test_write_mapping_info(): + return + + \ No newline at end of file diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 0ce8da3..74de958 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -8,12 +8,12 @@ import yaml import numpy as np +import pandas as pd import astropy.units as u from os.path import dirname, abspath, join, basename, isfile from astrort.utils.utils import * from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml -from rtasci.lib.RTAUtils import check_energy_thresholds from astropy.coordinates import SkyCoord def load_yaml_conf(yamlfile): @@ -89,4 +89,27 @@ def write_simulation_info(simulator, configuration, pointing, datfile, clock): f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf computation_time\n') with open(datfile, 'a') as f: f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {irf} {clock}\n') + +def merge_simulation_info(configuration, log): + folder = configuration['output'] + datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f] + merger = join(folder, 'merged_sim_data.dat') + # check merger file + if isfile(merger): + log.warning(f"Merger output already exists, overwrite {merger}") + with open(merger, 'w+') as f: + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf computation_time\n') + # collect data + for i, datfile in enumerate(datfiles): + log.info(f"Collect data from {datfile}") + data = pd.read_csv(join(datfile), sep=' ') + if i == 0: + table = data + else: + table = pd.concat([table, data], ignore_index=True) + log.info(f"Lines in data: {len(table)}") + # write merger file + table.to_csv(merger, index=False, header=True, sep=' ', na_rep=np.nan) + +def write_mapping_info(): return \ No newline at end of file -- GitLab From a9bc8f6e01076a6855e6372eac1d84e0e4ad1ccf Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 14:49:38 +0200 Subject: [PATCH 72/98] add mapper plot setting --- astrort/configure/check_configuration.py | 3 ++- astrort/configure/test.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index f6b721b..ffd50ff 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -67,11 +67,12 @@ class CheckConfiguration(): return self def check_mapper(self): - keys = ['exposure', 'smooth', 'center', 'region', 'output'] + keys = ['exposure', 'smooth', 'center', 'plot', 'region', 'output'] assert self.conf['mapper'].keys() == keys assert type(self.conf['mapper']['exposure']) == int assert type(self.conf['mapper']['smooth']) == (float or int) assert type(self.conf['mapper']['center']) in ['pointing', 'source'] + assert type(self.conf['mapper']['plot']) == bool assert type(self.conf['mapper']['region']) == bool assert type(self.conf['mapper']['output']) == str return self \ No newline at end of file diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index a13f31f..9e394c5 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -14,6 +14,7 @@ mapper: exposure: 10 smooth: 1 center: pointing + plot: false region: false output: /data01/homes/dipiano/astroRT/astrort/testing/tmp -- GitLab From 5d124d8a4cbe5a1c0d2d849781720adcead968a5 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 15:05:29 +0200 Subject: [PATCH 73/98] rename simulator logger with prefix --- astrort/simulator/base_simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index b41c240..f86ce53 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -18,7 +18,7 @@ from astrort.configure.slurmjobs import make_simulator_sbatch def base_simulator(configuration_file): clock = time() configuration = load_yaml_conf(configuration_file) - logfile = join(configuration['simulator']['output'], configuration['logging']['logfile']) + logfile = join(configuration['simulator']['output'], 'simulator_' + configuration['logging']['logfile']) datfile = logfile.replace('.log', '.dat') log = set_logger(get_log_level(configuration['logging']['level']), logfile) log.info(f"Simulator configured, took {time() - clock} s") -- GitLab From 25ac5fec514e6a26512b474ddea9a01b0b07c280 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 15:11:53 +0200 Subject: [PATCH 74/98] make simulator config not general conf --- astrort/configure/slurmjobs.py | 15 ++++++++++----- astrort/testing/test_configure/test_slurmjobs.py | 6 +++--- astrort/testing/test_utils/test_wrap.py | 4 ++-- astrort/utils/wrap.py | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index f5c58a2..20b8025 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -10,12 +10,12 @@ from yaml import dump from os import system from os.path import join, dirname, abspath -def make_configuration(jobname_conf, configuration, node_number): +def make_simulator_conf(jobname_conf, configuration, node_number): # simulator configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 # logging - configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.log') - configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}.dat') + configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.log') + configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) @@ -41,7 +41,12 @@ def make_simulator_sbatch(jobname, configuration, node_number): jobname_sh = join(output, f"{jobname}.sh") jobname_log = join(output, f"{jobname}.slurm") jobname_conf = join(output, f"{jobname}.yml") - make_configuration(jobname_conf, configuration, node_number) + make_simulator_conf(jobname_conf, configuration, node_number) make_simulator_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") - \ No newline at end of file + +def make_mapper_sh(): + return + +def make_mapper_sbatch(): + return \ No newline at end of file diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index c580685..d32bd0a 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -10,11 +10,11 @@ import pytest from shutil import rmtree from os import listdir, makedirs from os.path import isfile, join -from astrort.configure.slurmjobs import make_configuration, make_simulator_sh, make_simulator_sbatch +from astrort.configure.slurmjobs import make_simulator_conf, make_simulator_sh, make_simulator_sbatch from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file -def test_make_configuration(test_conf_file): +def test_make_simulator_conf(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) @@ -26,7 +26,7 @@ def test_make_configuration(test_conf_file): node_number += 1 jobname = f"{conf['slurm']['name']}_{node_number}" jobname_conf = join(conf['simulator']['output'], f"job_{jobname}.yml") - make_configuration(jobname_conf, conf, node_number) + make_simulator_conf(jobname_conf, conf, node_number) # check output expected_configurations = conf['slurm']['nodes'] diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 2df9d6e..22283bc 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -50,7 +50,7 @@ def test_write_simulation_info(test_conf_file): conf = load_yaml_conf(test_conf_file) conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} pointing = get_point_source_info(conf['simulator']) - datfile = join(conf['simulator']['output'], 'tmp.dat') + datfile = join(conf['simulator']['output'], 'simulator.dat') sim = RTACtoolsSimulation() clock = 1 write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) @@ -67,7 +67,7 @@ def test_merge_simulation_info(test_conf_file, test_tmp_folder): clock = 1 for i in range(5): sim.seed = i - datfile = join(conf['simulator']['output'], f'job_{i}.dat') + datfile = join(conf['simulator']['output'], f'job_{i}_simulator.dat') write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) assert isfile(datfile) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 74de958..025c4d6 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -92,7 +92,7 @@ def write_simulation_info(simulator, configuration, pointing, datfile, clock): def merge_simulation_info(configuration, log): folder = configuration['output'] - datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f] + datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f and 'simulator' in f] merger = join(folder, 'merged_sim_data.dat') # check merger file if isfile(merger): -- GitLab From 356a9ecac8ea75702e66bc6ace02f11016bf2f2b Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 15:18:53 +0200 Subject: [PATCH 75/98] get all seeds --- astrort/testing/test_utils/test_utils.py | 8 ++++++++ astrort/utils/utils.py | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 9c6d324..5db1d25 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -7,8 +7,10 @@ # ***************************************************************************** import pytest +import numpy as np from os.path import isdir from astrort.utils.utils import * +from astrort.utils.wrap import * @pytest.mark.test_tmp_folder @pytest.mark.parametrize('samples', [3, 5, 8, 10]) @@ -83,3 +85,9 @@ def test_select_irf(array): assert array in irf.lower() assert 'share/caldb/data/cta' in join(path, irf).lower() assert isdir(join(path, irf)) is True + +@pytest.mark.test_conf_file +def test_get_all_seeds(test_conf_file): + conf = load_yaml_conf(test_conf_file) + seeds = get_all_seeds(conf['simulator']) + assert seeds.all() == np.array([1, 2]).all() diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 95a2b66..76ff417 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -7,6 +7,7 @@ # ***************************************************************************** import random +import numpy as np from os import listdir from os.path import join, expandvars @@ -68,4 +69,10 @@ def select_random_irf(array, prod): path = join(expandvars('$CALDB'), f'data/cta/{prod}/bcf') irfs = listdir(path) irf = random.choice([i for i in irfs if array in i.lower()]) - return irf \ No newline at end of file + return irf + +def get_all_seeds(simulator): + start_seed = simulator['seed'] + samples = simulator['samples'] + seeds = np.arange(start_seed, samples+start_seed, step=1) + return seeds \ No newline at end of file -- GitLab From c091ad233cb33a601ef26768e2d34f4eb18d53da Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 25 Sep 2023 17:31:14 +0200 Subject: [PATCH 76/98] one make_sh with mode for operation --- astrort/configure/slurmjobs.py | 17 +++++---- .../testing/test_configure/test_slurmjobs.py | 6 ++-- .../test_simulator/test_base_mapper.py | 36 +++++++++++++++++++ 3 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 astrort/testing/test_simulator/test_base_mapper.py diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 20b8025..ed45e49 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -20,7 +20,7 @@ def make_simulator_conf(jobname_conf, configuration, node_number): with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -def make_simulator_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log): +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode='simulator'): # write sbatch with open(jobname_sh, 'w+') as f: f.write("#!/bin/bash") @@ -34,15 +34,18 @@ def make_simulator_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log) f.write(f"\n#SBATCH --partition={slurmconf['partition']}") f.write(f"\n") f.write(f"\nsource activate {slurmconf['environment']}") - f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") + if mode == 'simulator': + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") + else: + raise ValueError(f"Invalid 'mode' {mode}") def make_simulator_sbatch(jobname, configuration, node_number): output = configuration['simulator']['output'] - jobname_sh = join(output, f"{jobname}.sh") - jobname_log = join(output, f"{jobname}.slurm") - jobname_conf = join(output, f"{jobname}.yml") - make_simulator_conf(jobname_conf, configuration, node_number) - make_simulator_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) + jobname_sh = join(output, f"{jobname}_simulator.sh") + jobname_log = join(output, f"{jobname}_simulator.slurm") + jobname_conf = join(output, f"{jobname}_simulator.yml") + make_simulator_conf(jobname_conf, configuration, node_number, mode='simulator') + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") def make_mapper_sh(): diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index d32bd0a..fb21879 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -10,7 +10,7 @@ import pytest from shutil import rmtree from os import listdir, makedirs from os.path import isfile, join -from astrort.configure.slurmjobs import make_simulator_conf, make_simulator_sh, make_simulator_sbatch +from astrort.configure.slurmjobs import make_simulator_conf, make_sh, make_simulator_sbatch from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file @@ -34,7 +34,7 @@ def test_make_simulator_conf(test_conf_file): assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" @pytest.mark.test_conf_file -def test_make_simulator_sh(test_conf_file): +def test_make_sh(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) @@ -49,7 +49,7 @@ def test_make_simulator_sh(test_conf_file): jobname_sh = join(output, f"job_{jobname}.sh") jobname_log = join(output, f"job_{jobname}.log") jobname_conf = join(output, f"job_{jobname}.yml") - make_simulator_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log) + make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log, mode='simulator') # check output expected_sh = conf['slurm']['nodes'] diff --git a/astrort/testing/test_simulator/test_base_mapper.py b/astrort/testing/test_simulator/test_base_mapper.py new file mode 100644 index 0000000..4eb687c --- /dev/null +++ b/astrort/testing/test_simulator/test_base_mapper.py @@ -0,0 +1,36 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import pytest +from shutil import rmtree +from os import listdir +from os.path import isfile, join +from astrort.simulator.base_simulator import base_simulator +from astrort.simulator.base_mapper import base_mapper +from astrort.utils.wrap import load_yaml_conf + +@pytest.mark.skip('#TODO') +@pytest.mark.test_conf_file +@pytest.mark.parametrize('seeds', [None, list([1,2])]) +def test_base_mapper(test_conf_file, seeds): + + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['mapper']['output']) + + # run simulator + base_simulator(test_conf_file) + base_mapper(test_conf_file, seeds) + + # check output + expected_maps = conf['simulator']['samples'] + found_maps = len([f for f in listdir(conf['mapper']['output']) if isfile(join(conf['mapper']['output'], f)) and '.fits' in f and conf['mapper']['name'] in f]) + assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" + + + -- GitLab From f4803c579686aaf04053dd0654f7ba939e8bddef Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:01:31 +0200 Subject: [PATCH 77/98] map template --- astrort/testing/test_utils/test_utils.py | 3 +++ astrort/utils/utils.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/astrort/testing/test_utils/test_utils.py b/astrort/testing/test_utils/test_utils.py index 5db1d25..8187a47 100644 --- a/astrort/testing/test_utils/test_utils.py +++ b/astrort/testing/test_utils/test_utils.py @@ -91,3 +91,6 @@ def test_get_all_seeds(test_conf_file): conf = load_yaml_conf(test_conf_file) seeds = get_all_seeds(conf['simulator']) assert seeds.all() == np.array([1, 2]).all() + +def test_map_template(): + assert map_template() == join(dirname(abspath(__file__)).replace('testing/test_utils', 'templates'), 'base_empty_map.fits') \ No newline at end of file diff --git a/astrort/utils/utils.py b/astrort/utils/utils.py index 76ff417..4b3a8d9 100644 --- a/astrort/utils/utils.py +++ b/astrort/utils/utils.py @@ -9,9 +9,12 @@ import random import numpy as np from os import listdir -from os.path import join, expandvars +from os.path import join, expandvars, dirname, abspath -def seeds_to_string_formatter_files(samples, output, name, seed, ext): +def map_template(): + return join(dirname(abspath(__file__)).replace('utils', 'templates'), 'base_empty_map.fits') + +def seeds_to_string_formatter_files(samples, output, name, seed, ext, suffix=None): if samples <= 1e3: name = join(output, f"{name}_{seed:03d}.{ext}") elif samples <= 1e5: @@ -20,6 +23,9 @@ def seeds_to_string_formatter_files(samples, output, name, seed, ext): name = join(output, f"{name}_{seed:08d}.{ext}") else: name = join(output, f"{name}_{seed}.{ext}") + # suffix + if suffix is not None: + name.replace(f'.{ext}', f'_{suffix}.{ext}') return name def seeds_to_string_formatter(samples, name, seed): -- GitLab From dcfcd042bfbc116e78687382d0423c210768f739 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:01:46 +0200 Subject: [PATCH 78/98] move down increment seed --- astrort/simulator/base_simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index f86ce53..3beff65 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -36,11 +36,11 @@ def base_simulator(configuration_file): simulator = configure_simulator_no_visibility(simulator, configuration['simulator'], log) simulator.run_simulation() log.info(f"Simulation (seed = {configuration['simulator']['seed']}) complete, took {time() - clock_sim} s") - configuration['simulator']['seed'] += 1 # timing simulation clock_sim = time() - clock_sim # save simulation data write_simulation_info(simulator, configuration['simulator'], point, datfile, clock_sim) + configuration['simulator']['seed'] += 1 del simulator # end simulations log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") -- GitLab From 3a5826685fea4ab487f474b9d1912283aebc36c7 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:17:38 +0200 Subject: [PATCH 79/98] add pixelsize for mapper --- astrort/configure/check_configuration.py | 3 ++- astrort/configure/test.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/astrort/configure/check_configuration.py b/astrort/configure/check_configuration.py index ffd50ff..f06f205 100644 --- a/astrort/configure/check_configuration.py +++ b/astrort/configure/check_configuration.py @@ -67,10 +67,11 @@ class CheckConfiguration(): return self def check_mapper(self): - keys = ['exposure', 'smooth', 'center', 'plot', 'region', 'output'] + keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output'] assert self.conf['mapper'].keys() == keys assert type(self.conf['mapper']['exposure']) == int assert type(self.conf['mapper']['smooth']) == (float or int) + assert type(self.conf['mapper']['pixelsize']) == (float or int) assert type(self.conf['mapper']['center']) in ['pointing', 'source'] assert type(self.conf['mapper']['plot']) == bool assert type(self.conf['mapper']['region']) == bool diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 9e394c5..7f406e9 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -13,6 +13,7 @@ simulator: mapper: exposure: 10 smooth: 1 + pixelsize: 0.02 center: pointing plot: false region: false -- GitLab From f90bebc0da4a00271d3f06420284a9b47b6f18df Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:35:01 +0200 Subject: [PATCH 80/98] execute mapper and write mapper info --- astrort/testing/test_utils/test_wrap.py | 40 +++++++++++++++++++++++-- astrort/utils/wrap.py | 27 ++++++++++++++--- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 22283bc..424a747 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -9,8 +9,11 @@ import pytest import logging import numpy as np +from shutil import rmtree from astrort.utils.wrap import * +from astrort.utils.utils import seeds_to_string_formatter_files from astrort.configure.logging import set_logger +from astrort.simulator.base_simulator import base_simulator from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation @pytest.mark.test_conf_file @@ -76,8 +79,39 @@ def test_merge_simulation_info(test_conf_file, test_tmp_folder): assert isfile(join(conf['simulator']['output'], 'merged_sim_data.dat')) del sim -def test_write_mapping_info(): - return +@pytest.mark.test_conf_file +def test_write_mapping_info(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + datfile = join(conf['simulator']['output'], 'simulator.dat') + clock = 1 + write_mapping_info(conf, datfile, clock) + assert isfile(datfile) +@pytest.mark.test_conf_file +def test_configure_simulator_no_visibility(test_conf_file): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + log = set_logger(logging.CRITICAL) + sim = RTACtoolsSimulation() + sim = configure_simulator_no_visibility(sim, conf['simulator'], log) + + assert sim.t == [0, conf['simulator']['duration']] + assert sim.seed == conf['simulator']['seed'] + assert sim.set_log is False + assert sim.caldb == conf['simulator']['prod'] + del sim + +@pytest.mark.test_conf_file +def test_execute_mapper_no_visibility(test_conf_file): + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['simulator']['output']) + conf['simulator']['samples'] + log = set_logger(logging.CRITICAL) - \ No newline at end of file + # run simulator + base_simulator(test_conf_file) + execute_mapper_no_visibility(conf, log) + mapfile = seeds_to_string_formatter_files(conf['simulator']['samples'], conf['simulator']['output'], conf['simulator']['name'], conf['simulator']['seed'], 'fits', suffix='map') + assert isfile(mapfile) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 025c4d6..2e0b5da 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -11,10 +11,11 @@ import numpy as np import pandas as pd import astropy.units as u from os.path import dirname, abspath, join, basename, isfile -from astrort.utils.utils import * -from astrort.configure.check_configuration import CheckConfiguration from rtasci.lib.RTAManageXml import ManageXml from astropy.coordinates import SkyCoord +from astrort.utils.utils import * +from astrort.configure.check_configuration import CheckConfiguration +from astrort.utils.mapping import Mapper def load_yaml_conf(yamlfile): with open(yamlfile) as f: @@ -22,6 +23,14 @@ def load_yaml_conf(yamlfile): CheckConfiguration(configuration=configuration) return configuration +def execute_mapper_no_visibility(configuration, log): + phlist = seeds_to_string_formatter_files(configuration['simulator']['samples'], configuration['simulator']['output'], configuration['simulator']['name'], configuration['simulator']['seed'], 'fits') + skymap = phlist.replace('.fits', '_map.fits') + maproi = get_instrument_fov(configuration['simulator']['array']) + mapper = Mapper(log) + mapper.get_countmap_in_fits(dl3_file=phlist, fitsname=skymap, template=map_template(), maproi=maproi, pixelsize=configuration['mapper']['pixelsize'], trange=[0, configuration['mapper']['exposure']], sigma=configuration['mapper']['smooth']) + del mapper + def configure_simulator_no_visibility(simulator, configuration, log): if '$TEMPLATES$' in configuration['model']: configuration['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(configuration['model'])) @@ -111,5 +120,15 @@ def merge_simulation_info(configuration, log): # write merger file table.to_csv(merger, index=False, header=True, sep=' ', na_rep=np.nan) -def write_mapping_info(): - return \ No newline at end of file +def write_mapping_info(configuration, datfile, clock): + name = seeds_to_string_formatter(configuration['simulator']['samples'], configuration['simulator']['name'], configuration['simulator']['seed']) + seed = configuration['simulator']['seed'] + exposure = configuration['mapper']['exposure'] + center_type = configuration['mapper']['center'] + pixelsize = configuration['mapper']['pixelsize'] + smooth = configuration['mapper']['smooth'] + if not isfile(datfile): + with open(datfile, 'w+') as f: + f.write('name seed exposure center_on pixelsize smooth computation_time\n') + with open(datfile, 'a') as f: + f.write(f'{name} {seed} {exposure} {center_type} {pixelsize} {smooth} {clock}\n') -- GitLab From dd64b7e7dcd11d3d966e410abd5f0aaa313acbc3 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:35:22 +0200 Subject: [PATCH 81/98] add map template empty --- astrort/templates/base_empty_map.fits | Bin 0 -> 2007360 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 astrort/templates/base_empty_map.fits diff --git a/astrort/templates/base_empty_map.fits b/astrort/templates/base_empty_map.fits new file mode 100644 index 0000000000000000000000000000000000000000..62ac4466c5e8b8648dacaa8288c82de9dad21090 GIT binary patch literal 2007360 zcmWIc^bPQFRZy^1zyd-P^cB)Fb5a#j@>7cylJoP@@{4j6O7a!lJVSyNic1pnQWA?& zP&7Juh6H#6DWq1Eq~;Z8=I5dK z-`U^S*VQjXK>-qH8g4nM6`4snsS2LCiRq~dAw`LK#Rd6AB?`fn#U-h^nhKyOPApN# zELKQKP0P$nO;O0qQ&4v-E-A{-%g?Pun3tHBq5u{y$fzvNOfFW}Q7FsLDa}n)Fg7>S zQ78ZzZ)|F5t&o(Nl$@WEYNcRgU|{H|=BTHqXKZe!r)O+x>7n2n5#s0Q>#AU@pdRcU z>FXGvprDQvI{FHEiMgqu*eJ=!EJpYdNeL>`IViw00(<-#=z-!l0G!T}^Ye>RGV>Bk zz-AYvrll68<|U^p6y#^-l_+Qwmn0UIWagzSq=C)D9>3(7=i=%Ug3o=<2$yB%B^RaU zrskC>B$g9Gny z`6Y?qP;$&m&q+nG7M0`U?-$_j1IiyDah&G)C6;8Cr7Gm)=cQ+sl%{}Fb#iJ>YH>+s zVva&VKHLr;#}Fj*%=FAnETJxR)iy9x&{u%C4`yCs34Y@gf*f63gCm1M`Pb7q2vky| zsav;lbu4Lb+9I)?ZMfy@gEaSZTKKrashit^Ko5_1*GGcuDi z6q1Wl6H8K46iPBuAvG16aS9=>KCZ#e{sFcM>dqmKU|H1k90CfZMQss6{i-JK_VAijDV^)+=hpE`ohdJFt7q6glP&2 z*vu=*%*AaU*nO^kE=0L6H7^Cf>l75g?t_>IcAS-g0dDt!%){$C1yFoA1_Z;>GbBu* zBqYB@7so2N<|GyrgJKn*Pa(Se{Xj0nT3;f}3&}4@%uz_s&reYRJ1;XYIj1xQRKBOA zCZ>ReAf_oO_;`l7f=tBH9zr)SC$lUSMK8<@WZE?%z&|)N$Q7@7t`!CO#id255Z57V z$6&j-I=VPxZx2D-=K}Lla(+<}v?xwXOfJbU!Y~UY;NflN;RZa(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7 zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R z7!85Z5Eu=C(GVC7fzc2c4S~@R7zLvtFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8Umvs zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71* zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@? z8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;6Cw1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$( zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n zMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU z1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$( zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtqq)hQMeDjE2By2#kinXb6mkz-S1J zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mk zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By z2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mg(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7 zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!83@Fd71*Aut*OqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71* zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@? z8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8Umvs zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?3PwXRZa(GVC7 zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7 zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7zLvt zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71* zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@? z8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8Umvs zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;6Cw1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU z1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$( zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n zMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ON zU^E0qLtqq)hQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mk zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By z2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD zjE2By2#kinXb6mg(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7 zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!83@Fd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@? z8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsKtu=t0IqW# A2mk;8 literal 0 HcmV?d00001 -- GitLab From b08c33c906a5dacd78326fc3711496350c7ff762 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:40:29 +0200 Subject: [PATCH 82/98] base mapper and test --- astrort/simulator/base_mapper.py | 65 +++++++++++++++++++ .../test_simulator/test_base_mapper.py | 4 +- 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 astrort/simulator/base_mapper.py diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py new file mode 100644 index 0000000..eddbab3 --- /dev/null +++ b/astrort/simulator/base_mapper.py @@ -0,0 +1,65 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +from time import time +from os import makedirs +from os.path import join +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility +from astrort.utils.utils import get_all_seeds +from astrort.configure.logging import set_logger, get_log_level +from astrort.configure.slurmjobs import make_mapper_sbatch + +def base_mapper(configuration_file, seeds=None): + clock = time() + configuration = load_yaml_conf(configuration_file) + logfile = join(configuration['mapper']['output'], 'mapper_' + configuration['logging']['logfile']) + datfile = logfile.replace('.log', '.dat') + # set logger + log = set_logger(get_log_level(configuration['logging']['level']), logfile) + # collect simulations to map + if seeds is None: + log.info(f"Mapping of all simulations found") + seeds = get_all_seeds(configuration['simulator']) + log.info(f"Mapper configured, took {time() - clock} s") + # create output dir + log.info(f"Output folder: {configuration['mapper']['output']}") + makedirs(configuration['mapper']['output'], exist_ok=True) + # start mapping + log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") + for seed in seeds: + clock_map = time() + # check pointing option + execute_mapper_no_visibility(configuration, log) + log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s") + # timing simulation + clock_map = time() - clock_map + # save simulation data + write_mapping_info(configuration, datfile, clock_map) + configuration['simulator']['seed'] += 1 + # end simulations + log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n") + log.info(f"Process complete, took {time() - clock} s") + +def slurm_submission(configuration_file, nodes): + return + + +def main(configuration, nodes): + if nodes == 0: + base_mapper()(configuration) + else: + slurm_submission(configuration, nodes) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' ) + args = parser.parse_args() + + main(args.configuration, args.nodes) \ No newline at end of file diff --git a/astrort/testing/test_simulator/test_base_mapper.py b/astrort/testing/test_simulator/test_base_mapper.py index 4eb687c..5af3bac 100644 --- a/astrort/testing/test_simulator/test_base_mapper.py +++ b/astrort/testing/test_simulator/test_base_mapper.py @@ -14,7 +14,6 @@ from astrort.simulator.base_simulator import base_simulator from astrort.simulator.base_mapper import base_mapper from astrort.utils.wrap import load_yaml_conf -@pytest.mark.skip('#TODO') @pytest.mark.test_conf_file @pytest.mark.parametrize('seeds', [None, list([1,2])]) def test_base_mapper(test_conf_file, seeds): @@ -29,8 +28,7 @@ def test_base_mapper(test_conf_file, seeds): # check output expected_maps = conf['simulator']['samples'] - found_maps = len([f for f in listdir(conf['mapper']['output']) if isfile(join(conf['mapper']['output'], f)) and '.fits' in f and conf['mapper']['name'] in f]) + found_maps = len([f for f in listdir(conf['mapper']['output']) if isfile(join(conf['mapper']['output'], f)) and '.fits' in f and conf['simulator']['name'] in f and 'map' in f]) assert found_maps == expected_maps, f"Expected {expected_maps} maps, found {found_maps}" - -- GitLab From 8f5350e0103387c0507083de85bf4ac112d34dd8 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Thu, 28 Sep 2023 16:46:29 +0200 Subject: [PATCH 83/98] draft mapper sbatch --- astrort/configure/slurmjobs.py | 9 ++++++++- astrort/simulator/base_mapper.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index ed45e49..a302302 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -51,5 +51,12 @@ def make_simulator_sbatch(jobname, configuration, node_number): def make_mapper_sh(): return -def make_mapper_sbatch(): +def make_mapper_sbatch(jobname, configuration, node_number): + output = configuration['simulator']['output'] + jobname_sh = join(output, f"{jobname}_simulator.sh") + jobname_log = join(output, f"{jobname}_simulator.slurm") + jobname_conf = join(output, f"{jobname}_simulator.yml") + make_simulator_conf(jobname_conf, configuration, node_number, mode='simulator') + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) + system(f"sbatch {jobname_sh}") return \ No newline at end of file diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py index eddbab3..3489150 100644 --- a/astrort/simulator/base_mapper.py +++ b/astrort/simulator/base_mapper.py @@ -47,6 +47,16 @@ def base_mapper(configuration_file, seeds=None): log.info(f"Process complete, took {time() - clock} s") def slurm_submission(configuration_file, nodes): + configuration = load_yaml_conf(configuration_file) + log = set_logger(get_log_level(configuration['logging']['level'])) + # create output dir + log.info(f"Creating {configuration['simulator']['output']}") + makedirs(configuration['simulator']['output'], exist_ok=True) + # sbatch jobs per each nodes + configuration['slurm']['nodes'] = nodes + for node_number in range(configuration['slurm']['nodes']): + jobname = f"{configuration['slurm']['name']}_{node_number+1}" + make_mapper_sbatch(jobname, configuration, node_number) return -- GitLab From 86ce702e7a5f932bafaa713f2b28f192d7151844 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 2 Oct 2023 15:01:35 +0200 Subject: [PATCH 84/98] rename back make_configuration --- astrort/configure/slurmjobs.py | 6 +++--- astrort/testing/test_configure/test_slurmjobs.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index a302302..9c215b4 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -10,7 +10,7 @@ from yaml import dump from os import system from os.path import join, dirname, abspath -def make_simulator_conf(jobname_conf, configuration, node_number): +def make_configuration(jobname_conf, configuration, node_number): # simulator configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 # logging @@ -44,7 +44,7 @@ def make_simulator_sbatch(jobname, configuration, node_number): jobname_sh = join(output, f"{jobname}_simulator.sh") jobname_log = join(output, f"{jobname}_simulator.slurm") jobname_conf = join(output, f"{jobname}_simulator.yml") - make_simulator_conf(jobname_conf, configuration, node_number, mode='simulator') + make_configuration(jobname_conf, configuration, node_number, mode='simulator') make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") @@ -56,7 +56,7 @@ def make_mapper_sbatch(jobname, configuration, node_number): jobname_sh = join(output, f"{jobname}_simulator.sh") jobname_log = join(output, f"{jobname}_simulator.slurm") jobname_conf = join(output, f"{jobname}_simulator.yml") - make_simulator_conf(jobname_conf, configuration, node_number, mode='simulator') + make_configuration(jobname_conf, configuration, node_number, mode='simulator') make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") return \ No newline at end of file diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index fb21879..a172d27 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -10,11 +10,11 @@ import pytest from shutil import rmtree from os import listdir, makedirs from os.path import isfile, join -from astrort.configure.slurmjobs import make_simulator_conf, make_sh, make_simulator_sbatch +from astrort.configure.slurmjobs import make_configuration, make_sh, make_simulator_sbatch from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file -def test_make_simulator_conf(test_conf_file): +def test_make_configuration(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) @@ -26,7 +26,7 @@ def test_make_simulator_conf(test_conf_file): node_number += 1 jobname = f"{conf['slurm']['name']}_{node_number}" jobname_conf = join(conf['simulator']['output'], f"job_{jobname}.yml") - make_simulator_conf(jobname_conf, conf, node_number) + make_configuration(jobname_conf, conf, node_number) # check output expected_configurations = conf['slurm']['nodes'] -- GitLab From d8ea673c292e085f071b92738eea3dc69e26594a Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 2 Oct 2023 15:04:40 +0200 Subject: [PATCH 85/98] make_mapper_sbatch and make_sh mapper mode --- astrort/configure/slurmjobs.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 9c215b4..0e416e5 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -36,6 +36,8 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode='sim f.write(f"\nsource activate {slurmconf['environment']}") if mode == 'simulator': f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n") + elif mode == 'mapper': + f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n") else: raise ValueError(f"Invalid 'mode' {mode}") @@ -48,15 +50,12 @@ def make_simulator_sbatch(jobname, configuration, node_number): make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") -def make_mapper_sh(): - return - def make_mapper_sbatch(jobname, configuration, node_number): - output = configuration['simulator']['output'] - jobname_sh = join(output, f"{jobname}_simulator.sh") - jobname_log = join(output, f"{jobname}_simulator.slurm") - jobname_conf = join(output, f"{jobname}_simulator.yml") - make_configuration(jobname_conf, configuration, node_number, mode='simulator') + output = configuration['mapper']['output'] + jobname_sh = join(output, f"{jobname}_mapper.sh") + jobname_log = join(output, f"{jobname}_mapper.slurm") + jobname_conf = join(output, f"{jobname}_mapper.yml") + make_configuration(jobname_conf, configuration, node_number, mode='mapper') make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") return \ No newline at end of file -- GitLab From 3d9eac84eacbee3f4f311fa6ad4e593eae3bd8e7 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 2 Oct 2023 15:16:53 +0200 Subject: [PATCH 86/98] unify make_sh and make_sbatch and slurm_submission --- astrort/configure/slurmjobs.py | 38 ++++++++++--------- astrort/simulator/base_mapper.py | 18 +-------- astrort/simulator/base_simulator.py | 18 +-------- .../testing/test_configure/test_slurmjobs.py | 17 +++++---- 4 files changed, 34 insertions(+), 57 deletions(-) diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 0e416e5..33bfd56 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -7,8 +7,10 @@ # ***************************************************************************** from yaml import dump -from os import system +from os import system, makedirs from os.path import join, dirname, abspath +from astrort.utils.wrap import load_yaml_conf +from astrort.configure.logging import set_logger, get_log_level def make_configuration(jobname_conf, configuration, node_number): # simulator @@ -41,21 +43,23 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode='sim else: raise ValueError(f"Invalid 'mode' {mode}") -def make_simulator_sbatch(jobname, configuration, node_number): - output = configuration['simulator']['output'] - jobname_sh = join(output, f"{jobname}_simulator.sh") - jobname_log = join(output, f"{jobname}_simulator.slurm") - jobname_conf = join(output, f"{jobname}_simulator.yml") - make_configuration(jobname_conf, configuration, node_number, mode='simulator') +def make_sbatch(jobname, configuration, node_number, mode): + output = configuration[mode]['output'] + jobname_sh = join(output, f"{jobname}_{mode}.sh") + jobname_log = join(output, f"{jobname}_{mode}.slurm") + jobname_conf = join(output, f"{jobname}_{mode}.yml") + make_configuration(jobname_conf, configuration, node_number, mode=mode) make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) system(f"sbatch {jobname_sh}") - -def make_mapper_sbatch(jobname, configuration, node_number): - output = configuration['mapper']['output'] - jobname_sh = join(output, f"{jobname}_mapper.sh") - jobname_log = join(output, f"{jobname}_mapper.slurm") - jobname_conf = join(output, f"{jobname}_mapper.yml") - make_configuration(jobname_conf, configuration, node_number, mode='mapper') - make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) - system(f"sbatch {jobname_sh}") - return \ No newline at end of file + +def slurm_submission(configuration_file, nodes, mode): + configuration = load_yaml_conf(configuration_file) + log = set_logger(get_log_level(configuration['logging']['level'])) + # create output dir + log.info(f"Creating {configuration['simulator']['output']}") + makedirs(configuration['simulator']['output'], exist_ok=True) + # sbatch jobs per each nodes + configuration['slurm']['nodes'] = nodes + for node_number in range(configuration['slurm']['nodes']): + jobname = f"{configuration['slurm']['name']}_{node_number+1}" + make_sbatch(jobname, configuration, node_number, mode=mode) diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py index 3489150..009de17 100644 --- a/astrort/simulator/base_mapper.py +++ b/astrort/simulator/base_mapper.py @@ -13,7 +13,7 @@ from os.path import join from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility from astrort.utils.utils import get_all_seeds from astrort.configure.logging import set_logger, get_log_level -from astrort.configure.slurmjobs import make_mapper_sbatch +from astrort.configure.slurmjobs import slurm_submission def base_mapper(configuration_file, seeds=None): clock = time() @@ -46,25 +46,11 @@ def base_mapper(configuration_file, seeds=None): log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n") log.info(f"Process complete, took {time() - clock} s") -def slurm_submission(configuration_file, nodes): - configuration = load_yaml_conf(configuration_file) - log = set_logger(get_log_level(configuration['logging']['level'])) - # create output dir - log.info(f"Creating {configuration['simulator']['output']}") - makedirs(configuration['simulator']['output'], exist_ok=True) - # sbatch jobs per each nodes - configuration['slurm']['nodes'] = nodes - for node_number in range(configuration['slurm']['nodes']): - jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_mapper_sbatch(jobname, configuration, node_number) - return - - def main(configuration, nodes): if nodes == 0: base_mapper()(configuration) else: - slurm_submission(configuration, nodes) + slurm_submission(configuration, nodes, mode='mapper') if __name__ == '__main__': parser = argparse.ArgumentParser(description='') diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 3beff65..7e14068 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -13,7 +13,7 @@ from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing from astrort.configure.logging import set_logger, get_log_level -from astrort.configure.slurmjobs import make_simulator_sbatch +from astrort.configure.slurmjobs import slurm_submission def base_simulator(configuration_file): clock = time() @@ -46,25 +46,11 @@ def base_simulator(configuration_file): log.info(f"\n {'-'*17} \n| STOP SIMULATOR | \n {'-'*17} \n") log.info(f"Process complete, took {time() - clock} s") - -def slurm_submission(configuration_file, nodes): - configuration = load_yaml_conf(configuration_file) - log = set_logger(get_log_level(configuration['logging']['level'])) - # create output dir - log.info(f"Creating {configuration['simulator']['output']}") - makedirs(configuration['simulator']['output'], exist_ok=True) - # sbatch jobs per each nodes - configuration['slurm']['nodes'] = nodes - for node_number in range(configuration['slurm']['nodes']): - jobname = f"{configuration['slurm']['name']}_{node_number+1}" - make_simulator_sbatch(jobname, configuration, node_number) - return - def main(configuration, nodes): if nodes == 0: base_simulator(configuration) else: - slurm_submission(configuration, nodes) + slurm_submission(configuration, nodes, mode='simulator') if __name__ == '__main__': parser = argparse.ArgumentParser(description='') diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index a172d27..1cc1e54 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -10,7 +10,7 @@ import pytest from shutil import rmtree from os import listdir, makedirs from os.path import isfile, join -from astrort.configure.slurmjobs import make_configuration, make_sh, make_simulator_sbatch +from astrort.configure.slurmjobs import make_configuration, make_sh from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file @@ -34,7 +34,8 @@ def test_make_configuration(test_conf_file): assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" @pytest.mark.test_conf_file -def test_make_sh(test_conf_file): +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_make_sh(test_conf_file, mode): # clean output conf = load_yaml_conf(test_conf_file) @@ -46,13 +47,13 @@ def test_make_sh(test_conf_file): for node_number in range(conf['slurm']['nodes']): node_number += 1 jobname = f"{conf['slurm']['name']}_{node_number}" - jobname_sh = join(output, f"job_{jobname}.sh") - jobname_log = join(output, f"job_{jobname}.log") - jobname_conf = join(output, f"job_{jobname}.yml") - make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log, mode='simulator') + jobname_sh = join(output, f"job_{jobname}_{mode}.sh") + jobname_log = join(output, f"job_{jobname}_{mode}.log") + jobname_conf = join(output, f"job_{jobname}_{mode}.yml") + make_sh(jobname, conf['slurm'], jobname_conf, jobname_sh, jobname_log, mode=mode) # check output expected_sh = conf['slurm']['nodes'] - found_sh = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.sh' in f and conf['slurm']['name'] in f]) - assert found_sh == expected_sh, f"Expected {expected_sh} simulations, found {found_sh}" + found_sh = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.sh' in f and conf['slurm']['name'] in f and mode in f]) + assert found_sh == expected_sh, f"Expected {expected_sh} files for {mode}, found {found_sh}" -- GitLab From 628d53d670a5e365245f442efe44ea029f768d62 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 2 Oct 2023 16:30:22 +0200 Subject: [PATCH 87/98] fix makedirs and rmtree and also logname --- astrort/configure/logging.py | 13 ++++++++---- astrort/configure/slurmjobs.py | 14 ++++++------- astrort/simulator/base_mapper.py | 6 +++--- astrort/simulator/base_simulator.py | 8 +++---- .../testing/test_configure/test_logging.py | 14 +++++++++++-- .../testing/test_configure/test_slurmjobs.py | 21 ++++++++++--------- .../test_simulator/test_base_mapper.py | 2 +- .../test_simulator/test_base_simulator.py | 2 +- astrort/testing/test_utils/test_wrap.py | 2 +- 9 files changed, 48 insertions(+), 34 deletions(-) diff --git a/astrort/configure/logging.py b/astrort/configure/logging.py index 37df7fd..55fcc49 100644 --- a/astrort/configure/logging.py +++ b/astrort/configure/logging.py @@ -8,7 +8,7 @@ import logging from os import makedirs -from os.path import isdir, dirname, isfile +from os.path import dirname, isfile, join def set_logger(level, filename=None): log = logging.getLogger() @@ -20,8 +20,7 @@ def set_logger(level, filename=None): log.addHandler(consoleHandler) # file logger if filename is not None: - if not isdir(dirname(filename)): - makedirs(dirname(filename)) + makedirs(dirname(filename), exist_ok=True) if not isfile(filename): f = open(filename, 'w+') f.close() @@ -45,4 +44,10 @@ def get_log_level(level): level = logging.NOTSET return level - return \ No newline at end of file +def get_logfile(configuration, mode): + logfile = configuration['logging']['logfile'] + if mode not in logfile: + logfile = logfile.replace('.log', f'_{mode}.log') + if configuration[mode]['output'] not in logfile: + logfile = join(configuration[mode]['output'], logfile) + return logfile \ No newline at end of file diff --git a/astrort/configure/slurmjobs.py b/astrort/configure/slurmjobs.py index 33bfd56..deb8651 100644 --- a/astrort/configure/slurmjobs.py +++ b/astrort/configure/slurmjobs.py @@ -12,17 +12,17 @@ from os.path import join, dirname, abspath from astrort.utils.wrap import load_yaml_conf from astrort.configure.logging import set_logger, get_log_level -def make_configuration(jobname_conf, configuration, node_number): +def make_configuration(jobname_conf, configuration, node_number, mode): # simulator configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1 # logging - configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.log') - configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.dat') + configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log') + configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat') # write new configuration with open(jobname_conf, 'w+') as f: dump(configuration, f, default_flow_style=False) -def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode='simulator'): +def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): # write sbatch with open(jobname_sh, 'w+') as f: f.write("#!/bin/bash") @@ -49,15 +49,15 @@ def make_sbatch(jobname, configuration, node_number, mode): jobname_log = join(output, f"{jobname}_{mode}.slurm") jobname_conf = join(output, f"{jobname}_{mode}.yml") make_configuration(jobname_conf, configuration, node_number, mode=mode) - make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log) + make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode) system(f"sbatch {jobname_sh}") def slurm_submission(configuration_file, nodes, mode): configuration = load_yaml_conf(configuration_file) log = set_logger(get_log_level(configuration['logging']['level'])) # create output dir - log.info(f"Creating {configuration['simulator']['output']}") - makedirs(configuration['simulator']['output'], exist_ok=True) + log.info(f"Creating {configuration[mode]['output']}") + makedirs(configuration[mode]['output'], exist_ok=True) # sbatch jobs per each nodes configuration['slurm']['nodes'] = nodes for node_number in range(configuration['slurm']['nodes']): diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py index 009de17..3be803a 100644 --- a/astrort/simulator/base_mapper.py +++ b/astrort/simulator/base_mapper.py @@ -12,13 +12,13 @@ from os import makedirs from os.path import join from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility from astrort.utils.utils import get_all_seeds -from astrort.configure.logging import set_logger, get_log_level +from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission def base_mapper(configuration_file, seeds=None): clock = time() configuration = load_yaml_conf(configuration_file) - logfile = join(configuration['mapper']['output'], 'mapper_' + configuration['logging']['logfile']) + logfile = get_logfile(configuration, mode='mapper') datfile = logfile.replace('.log', '.dat') # set logger log = set_logger(get_log_level(configuration['logging']['level']), logfile) @@ -48,7 +48,7 @@ def base_mapper(configuration_file, seeds=None): def main(configuration, nodes): if nodes == 0: - base_mapper()(configuration) + base_mapper(configuration) else: slurm_submission(configuration, nodes, mode='mapper') diff --git a/astrort/simulator/base_simulator.py b/astrort/simulator/base_simulator.py index 7e14068..9a1878b 100644 --- a/astrort/simulator/base_simulator.py +++ b/astrort/simulator/base_simulator.py @@ -8,23 +8,21 @@ import argparse from time import time -from os import makedirs -from os.path import join from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing -from astrort.configure.logging import set_logger, get_log_level +from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission def base_simulator(configuration_file): clock = time() configuration = load_yaml_conf(configuration_file) - logfile = join(configuration['simulator']['output'], 'simulator_' + configuration['logging']['logfile']) + logfile = get_logfile(configuration, mode='simulator') datfile = logfile.replace('.log', '.dat') log = set_logger(get_log_level(configuration['logging']['level']), logfile) log.info(f"Simulator configured, took {time() - clock} s") # create output dir log.info(f"Output folder: {configuration['simulator']['output']}") - makedirs(configuration['simulator']['output'], exist_ok=True) + #makedirs(configuration['simulator']['output'], exist_ok=True) # start simulations log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n") for i in range(configuration['simulator']['samples']): diff --git a/astrort/testing/test_configure/test_logging.py b/astrort/testing/test_configure/test_logging.py index 30a6c0d..8567b82 100644 --- a/astrort/testing/test_configure/test_logging.py +++ b/astrort/testing/test_configure/test_logging.py @@ -9,7 +9,8 @@ import pytest import logging from os.path import join, isfile -from astrort.configure.logging import get_log_level, set_logger +from astrort.configure.logging import * +from astrort.utils.wrap import load_yaml_conf def test_set_logger(): assert get_log_level('DEBUG') == logging.DEBUG @@ -19,8 +20,17 @@ def test_set_logger(): assert get_log_level('CRITICAL') == logging.CRITICAL @pytest.mark.test_tmp_folder -@pytest.mark.test_conf_file def test_set_logger(test_tmp_folder): log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) log.debug('TEST') assert isfile(join(test_tmp_folder, 'test_set_logger.log')) is True + +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_get_logfile(test_conf_file, mode): + conf = load_yaml_conf(test_conf_file) + + logfile = get_logfile(conf, mode) + assert mode in logfile + + diff --git a/astrort/testing/test_configure/test_slurmjobs.py b/astrort/testing/test_configure/test_slurmjobs.py index 1cc1e54..dc6ccb1 100644 --- a/astrort/testing/test_configure/test_slurmjobs.py +++ b/astrort/testing/test_configure/test_slurmjobs.py @@ -14,23 +14,24 @@ from astrort.configure.slurmjobs import make_configuration, make_sh from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file -def test_make_configuration(test_conf_file): +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_make_configuration(test_conf_file, mode): # clean output conf = load_yaml_conf(test_conf_file) - rmtree(conf['simulator']['output']) - makedirs(conf['simulator']['output'], exist_ok=True) + rmtree(conf[mode]['output']) + makedirs(conf[mode]['output'], exist_ok=True) # make configurations for node_number in range(conf['slurm']['nodes']): node_number += 1 jobname = f"{conf['slurm']['name']}_{node_number}" - jobname_conf = join(conf['simulator']['output'], f"job_{jobname}.yml") - make_configuration(jobname_conf, conf, node_number) + jobname_conf = join(conf[mode]['output'], f"job_{jobname}_{mode}.yml") + make_configuration(jobname_conf, conf, node_number, mode) # check output expected_configurations = conf['slurm']['nodes'] - found_configurations = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.yml' in f and conf['slurm']['name'] in f]) + found_configurations = len([f for f in listdir(conf[mode]['output']) if isfile(join(conf[mode]['output'], f)) and '.yml' in f and conf['slurm']['name'] in f]) assert found_configurations == expected_configurations, f"Expected {expected_configurations} simulations, found {found_configurations}" @pytest.mark.test_conf_file @@ -39,11 +40,11 @@ def test_make_sh(test_conf_file, mode): # clean output conf = load_yaml_conf(test_conf_file) - rmtree(conf['simulator']['output']) - makedirs(conf['simulator']['output'], exist_ok=True) + rmtree(conf[mode]['output']) + makedirs(conf[mode]['output'], exist_ok=True) # make configurations - output = conf['simulator']['output'] + output = conf[mode]['output'] for node_number in range(conf['slurm']['nodes']): node_number += 1 jobname = f"{conf['slurm']['name']}_{node_number}" @@ -54,6 +55,6 @@ def test_make_sh(test_conf_file, mode): # check output expected_sh = conf['slurm']['nodes'] - found_sh = len([f for f in listdir(conf['simulator']['output']) if isfile(join(conf['simulator']['output'], f)) and '.sh' in f and conf['slurm']['name'] in f and mode in f]) + found_sh = len([f for f in listdir(conf[mode]['output']) if isfile(join(conf[mode]['output'], f)) and '.sh' in f and conf['slurm']['name'] in f and mode in f]) assert found_sh == expected_sh, f"Expected {expected_sh} files for {mode}, found {found_sh}" diff --git a/astrort/testing/test_simulator/test_base_mapper.py b/astrort/testing/test_simulator/test_base_mapper.py index 5af3bac..cda6fad 100644 --- a/astrort/testing/test_simulator/test_base_mapper.py +++ b/astrort/testing/test_simulator/test_base_mapper.py @@ -20,7 +20,7 @@ def test_base_mapper(test_conf_file, seeds): # clean output conf = load_yaml_conf(test_conf_file) - rmtree(conf['mapper']['output']) + rmtree(conf['mapper']['output'], ignore_errors=True) # run simulator base_simulator(test_conf_file) diff --git a/astrort/testing/test_simulator/test_base_simulator.py b/astrort/testing/test_simulator/test_base_simulator.py index 4c35fcb..fc716e7 100644 --- a/astrort/testing/test_simulator/test_base_simulator.py +++ b/astrort/testing/test_simulator/test_base_simulator.py @@ -18,7 +18,7 @@ def test_base_simulator(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) - rmtree(conf['simulator']['output']) + rmtree(conf['simulator']['output'], ignore_errors=True) # run simulator base_simulator(test_conf_file) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 424a747..65a49f8 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -106,7 +106,7 @@ def test_configure_simulator_no_visibility(test_conf_file): def test_execute_mapper_no_visibility(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) - rmtree(conf['simulator']['output']) + rmtree(conf['mapper']['output'], ignore_errors=True) conf['simulator']['samples'] log = set_logger(logging.CRITICAL) -- GitLab From a7ff6274fb8fb1b2e445694685820d747fc30ccd Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 2 Oct 2023 17:57:51 +0200 Subject: [PATCH 88/98] plotting --- astrort/configure/test.yml | 2 +- astrort/simulator/base_mapper.py | 12 +++++++---- .../test_simulator/test_base_mapper.py | 5 ++++- astrort/testing/test_utils/test_wrap.py | 21 +++++++++++++++---- astrort/utils/plotting.py | 9 ++++---- astrort/utils/wrap.py | 10 +++++++++ 6 files changed, 44 insertions(+), 15 deletions(-) diff --git a/astrort/configure/test.yml b/astrort/configure/test.yml index 7f406e9..645c041 100644 --- a/astrort/configure/test.yml +++ b/astrort/configure/test.yml @@ -15,7 +15,7 @@ mapper: smooth: 1 pixelsize: 0.02 center: pointing - plot: false + plot: true region: false output: /data01/homes/dipiano/astroRT/astrort/testing/tmp diff --git a/astrort/simulator/base_mapper.py b/astrort/simulator/base_mapper.py index 3be803a..bebc4e2 100644 --- a/astrort/simulator/base_mapper.py +++ b/astrort/simulator/base_mapper.py @@ -9,8 +9,7 @@ import argparse from time import time from os import makedirs -from os.path import join -from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility +from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility, plot_map from astrort.utils.utils import get_all_seeds from astrort.configure.logging import set_logger, get_log_level, get_logfile from astrort.configure.slurmjobs import slurm_submission @@ -34,9 +33,14 @@ def base_mapper(configuration_file, seeds=None): log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n") for seed in seeds: clock_map = time() - # check pointing option - execute_mapper_no_visibility(configuration, log) + # make map + fitsmap = execute_mapper_no_visibility(configuration, log) log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s") + # make plot + if configuration['mapper']['plot']: + clock_plot = time() + plotmap = plot_map(fitsmap, log) + log.info(f"Plotting (seed = {seed}) complete, took {time() - clock_plot} s") # timing simulation clock_map = time() - clock_map # save simulation data diff --git a/astrort/testing/test_simulator/test_base_mapper.py b/astrort/testing/test_simulator/test_base_mapper.py index cda6fad..f0df454 100644 --- a/astrort/testing/test_simulator/test_base_mapper.py +++ b/astrort/testing/test_simulator/test_base_mapper.py @@ -7,6 +7,7 @@ # ***************************************************************************** import pytest +import numpy as np from shutil import rmtree from os import listdir from os.path import isfile, join @@ -15,7 +16,7 @@ from astrort.simulator.base_mapper import base_mapper from astrort.utils.wrap import load_yaml_conf @pytest.mark.test_conf_file -@pytest.mark.parametrize('seeds', [None, list([1,2])]) +@pytest.mark.parametrize('seeds', [None, list()]) def test_base_mapper(test_conf_file, seeds): # clean output @@ -24,6 +25,8 @@ def test_base_mapper(test_conf_file, seeds): # run simulator base_simulator(test_conf_file) + if type(seeds) == list: + seeds = np.arange(conf['simulator']['samples']) base_mapper(test_conf_file, seeds) # check output diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 65a49f8..1736e45 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -107,11 +107,24 @@ def test_execute_mapper_no_visibility(test_conf_file): # clean output conf = load_yaml_conf(test_conf_file) rmtree(conf['mapper']['output'], ignore_errors=True) - conf['simulator']['samples'] + conf['simulator']['samples'] = 1 log = set_logger(logging.CRITICAL) # run simulator base_simulator(test_conf_file) - execute_mapper_no_visibility(conf, log) - mapfile = seeds_to_string_formatter_files(conf['simulator']['samples'], conf['simulator']['output'], conf['simulator']['name'], conf['simulator']['seed'], 'fits', suffix='map') - assert isfile(mapfile) + fitsmap = execute_mapper_no_visibility(conf, log) + assert isfile(fitsmap) + +@pytest.mark.test_conf_file +def test_plot_map(test_conf_file): + # clean output + conf = load_yaml_conf(test_conf_file) + rmtree(conf['mapper']['output'], ignore_errors=True) + conf['simulator']['samples'] = 1 + log = set_logger(logging.CRITICAL) + + # run simulator + base_simulator(test_conf_file) + fitsmap = execute_mapper_no_visibility(conf, log) + plotmap = plot_map(fitsmap, log) + assert isfile(plotmap) \ No newline at end of file diff --git a/astrort/utils/plotting.py b/astrort/utils/plotting.py index 9472805..3d12d46 100644 --- a/astrort/utils/plotting.py +++ b/astrort/utils/plotting.py @@ -102,7 +102,6 @@ class Plotter(): return ra, dec def counts_map(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): - self.__check_pointing() extent = self.get_extent(roi) # counts map data = self.get_phlist_data(file=file) @@ -129,7 +128,8 @@ class Plotter(): ax.set_ylabel(ylabel, fontsize=fontsize) ax.set_title(title, fontsize=fontsize) # pointing - ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + if isinstance(self.pointing, dict): + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) ax.set_xlim((extent[0], extent[1])) ax.set_ylim((extent[2], extent[3])) ax.set_aspect('equal') @@ -141,7 +141,6 @@ class Plotter(): return self def counts_map_with_wcs(self, file, trange=None, erange=None, roi=5, name='skymap.png', title=None, xlabel='right ascension (deg)', ylabel='declination (deg)', figsize=(10, 10), fontsize=20, cmap='CMRmap', pixelsize=0.02, sigma=1): - self.__check_pointing() extent = self.get_extent(roi=roi) # counts map data = self.get_phlist_data(file=file) @@ -172,7 +171,8 @@ class Plotter(): ax.set_ylabel(ylabel, fontsize=fontsize) ax.set_title(title, fontsize=fontsize) # pointing - ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) + if isinstance(self.pointing, dict): + ax.plot([self.pointing['ra']], [self.pointing['dec']], 'k+', markersize=fontsize) ax.set_aspect('equal') #ax.invert_xaxis() ax.grid(color='grey', ls='solid') @@ -214,7 +214,6 @@ class Plotter(): with fits.open(file) as h: try: w = WCS(h['SKYMAP'].header) - print(h['SKYMAP'].header) data = h['SKYMAP'].data except KeyError as e: self.log.error(f'Missing "SKYMAP" extention, the input file may not be a compatible counts map. {e}') diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 2e0b5da..5709438 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -16,6 +16,7 @@ from astropy.coordinates import SkyCoord from astrort.utils.utils import * from astrort.configure.check_configuration import CheckConfiguration from astrort.utils.mapping import Mapper +from astrort.utils.plotting import Plotter def load_yaml_conf(yamlfile): with open(yamlfile) as f: @@ -30,6 +31,7 @@ def execute_mapper_no_visibility(configuration, log): mapper = Mapper(log) mapper.get_countmap_in_fits(dl3_file=phlist, fitsname=skymap, template=map_template(), maproi=maproi, pixelsize=configuration['mapper']['pixelsize'], trange=[0, configuration['mapper']['exposure']], sigma=configuration['mapper']['smooth']) del mapper + return skymap def configure_simulator_no_visibility(simulator, configuration, log): if '$TEMPLATES$' in configuration['model']: @@ -132,3 +134,11 @@ def write_mapping_info(configuration, datfile, clock): f.write('name seed exposure center_on pixelsize smooth computation_time\n') with open(datfile, 'a') as f: f.write(f'{name} {seed} {exposure} {center_type} {pixelsize} {smooth} {clock}\n') + +def plot_map(fitsmap, log): + plotmap = fitsmap.replace('.fits', '.png') + plot = Plotter(log) + plot.plot_fits_skymap(fitsmap, plotmap) + del plot + return plotmap + \ No newline at end of file -- GitLab From a44f871b276262cee27707c1081b9abfec5ab393 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 10 Oct 2023 13:38:58 +0200 Subject: [PATCH 89/98] fix mapper output path --- astrort/utils/wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 5709438..3ddd418 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -26,7 +26,7 @@ def load_yaml_conf(yamlfile): def execute_mapper_no_visibility(configuration, log): phlist = seeds_to_string_formatter_files(configuration['simulator']['samples'], configuration['simulator']['output'], configuration['simulator']['name'], configuration['simulator']['seed'], 'fits') - skymap = phlist.replace('.fits', '_map.fits') + skymap = phlist.replace('.fits', '_map.fits').replace(configuration['simulator']['output'], configuration['mapper']['output']) maproi = get_instrument_fov(configuration['simulator']['array']) mapper = Mapper(log) mapper.get_countmap_in_fits(dl3_file=phlist, fitsname=skymap, template=map_template(), maproi=maproi, pixelsize=configuration['mapper']['pixelsize'], trange=[0, configuration['mapper']['exposure']], sigma=configuration['mapper']['smooth']) -- GitLab From e5c954203e971626ab620efe14333f8cfa29ce87 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 10 Oct 2023 15:41:47 +0200 Subject: [PATCH 90/98] background model --- astrort/templates/background.xml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 astrort/templates/background.xml diff --git a/astrort/templates/background.xml b/astrort/templates/background.xml new file mode 100644 index 0000000..2a12e95 --- /dev/null +++ b/astrort/templates/background.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file -- GitLab From aa42d1ecf3ddca77ace219b0c6db84b1498dd080 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 10 Oct 2023 15:44:54 +0200 Subject: [PATCH 91/98] fix background model --- astrort/templates/background.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/templates/background.xml b/astrort/templates/background.xml index 2a12e95..3cf3b74 100644 --- a/astrort/templates/background.xml +++ b/astrort/templates/background.xml @@ -1,4 +1,5 @@ + -- GitLab From c00bbf62de61573fb51967e75767f864cf92d70f Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Tue, 10 Oct 2023 16:30:53 +0200 Subject: [PATCH 92/98] sim backgrounds --- astrort/testing/test_utils/test_wrap.py | 27 ++++++++++++++++------ astrort/utils/wrap.py | 30 +++++++++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 1736e45..0359ddf 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -22,8 +22,11 @@ def test_load_yaml_conf(test_conf_file): assert type(configuration) == dict @pytest.mark.test_conf_file -def test_randomise_pointing_sim(test_conf_file): +@pytest.mark.parametrize('model', ['crab.xml', 'background.xml']) +def test_randomise_pointing_sim(test_conf_file, model): conf = load_yaml_conf(test_conf_file) + conf['simulator']['name'] = model.replace('.xml', '') + conf['simulator']['model'] = f'$TEMPLATES$/{model}' pointing = randomise_pointing_sim(conf['simulator']) assert type(pointing) == dict for key in pointing.keys(): @@ -35,18 +38,28 @@ def test_randomise_pointing_sim(test_conf_file): assert type(pointing['source_dec']) == type(np.float64(1)) @pytest.mark.test_conf_file -def test_get_point_source_info(test_conf_file): +@pytest.mark.parametrize('model', ['crab.xml', 'background.xml']) +def test_get_point_source_info(test_conf_file, model): conf = load_yaml_conf(test_conf_file) + conf['simulator']['name'] = model.replace('.xml', '') + conf['simulator']['model'] = f'$TEMPLATES$/{model}' conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} pointing = get_point_source_info(conf['simulator']) assert type(pointing) == dict for key in pointing.keys(): assert key in ['point_ra', 'point_dec', 'offset', 'source_ra', 'source_dec'] - assert type(pointing['point_ra']) == type(np.float64(1)) - assert type(pointing['point_dec']) == type(np.float64(1)) - assert type(pointing['offset']) == type(np.float64(1)) - assert type(pointing['source_ra']) == type(np.float64(1)) - assert type(pointing['source_dec']) == type(np.float64(1)) + if 'background.xml' not in model: + assert type(pointing['point_ra']) == type(np.float64(1)) + assert type(pointing['point_dec']) == type(np.float64(1)) + assert type(pointing['offset']) == type(np.float64(1)) + assert type(pointing['source_ra']) == type(np.float64(1)) + assert type(pointing['source_dec']) == type(np.float64(1)) + else: + assert type(pointing['point_ra']) == type(1) + assert type(pointing['point_dec']) == type(1) + assert type(pointing['offset']) == type(np.nan) + assert type(pointing['source_ra']) == type(np.nan) + assert type(pointing['source_dec']) == type(np.nan) @pytest.mark.test_conf_file def test_write_simulation_info(test_conf_file): diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 3ddd418..79ee504 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -65,11 +65,14 @@ def set_pointing(simulator, configuration, log): def randomise_pointing_sim(simulator): if '$TEMPLATES$' in simulator['model']: simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) - model_xml = ManageXml(xml=simulator['model']) - source = model_xml.getRaDec() - del model_xml + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) + source = model_xml.getRaDec() + del model_xml + ra, dec = source[0][0] * u.deg, source[1][0] * u.deg + else: + ra, dec = np.random.uniform(0, 360) * u.deg, np.random.uniform(-90, 90) * u.deg # use astropy separation - ra, dec = source[0][0] * u.deg, source[1][0] * u.deg source = SkyCoord(ra, dec, frame='icrs') position_angle = 45 * u.deg separation = np.random.random() * get_instrument_fov(simulator['array']) * u.deg @@ -79,14 +82,17 @@ def randomise_pointing_sim(simulator): def get_point_source_info(simulator): if '$TEMPLATES$' in simulator['model']: simulator['model'] = join(dirname(abspath(__file__)).replace('utils', 'templates'), basename(simulator['model'])) - model_xml = ManageXml(xml=simulator['model']) - source = model_xml.getRaDec() - del model_xml - # use astropy separation - source = SkyCoord(source[0][0] * u.deg, source[1][0] * u.deg, frame='icrs') - pointing = SkyCoord(simulator['pointing']['ra'] * u.deg, simulator['pointing']['dec'] * u.deg, frame='icrs') - separation = source.separation(pointing) - return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + if 'background.xml' not in simulator['model']: + model_xml = ManageXml(xml=simulator['model']) + source = model_xml.getRaDec() + del model_xml + # use astropy separation + source = SkyCoord(source[0][0] * u.deg, source[1][0] * u.deg, frame='icrs') + pointing = SkyCoord(simulator['pointing']['ra'] * u.deg, simulator['pointing']['dec'] * u.deg, frame='icrs') + separation = source.separation(pointing) + return {'point_ra': pointing.ra.deg, 'point_dec': pointing.dec.deg, 'offset': separation.value, 'source_ra': source.ra.deg, 'source_dec': source.dec.deg} + else: + return {'point_ra': simulator['pointing']['ra'], 'point_dec': simulator['pointing']['dec'], 'offset': np.nan, 'source_ra': np.nan, 'source_dec': np.nan} def write_simulation_info(simulator, configuration, pointing, datfile, clock): name = seeds_to_string_formatter(configuration['samples'], configuration['name'], configuration['seed']) -- GitLab From 82c9cddf622843615ee21b171a2aaa6f70891d65 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 18 Oct 2023 11:31:20 +0200 Subject: [PATCH 93/98] add merge_data_info and test --- astrort/testing/test_utils/test_wrap.py | 27 +++++++++++++++++++++++-- astrort/utils/wrap.py | 21 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/astrort/testing/test_utils/test_wrap.py b/astrort/testing/test_utils/test_wrap.py index 0359ddf..e4a9e8a 100644 --- a/astrort/testing/test_utils/test_wrap.py +++ b/astrort/testing/test_utils/test_wrap.py @@ -11,7 +11,6 @@ import logging import numpy as np from shutil import rmtree from astrort.utils.wrap import * -from astrort.utils.utils import seeds_to_string_formatter_files from astrort.configure.logging import set_logger from astrort.simulator.base_simulator import base_simulator from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation @@ -92,11 +91,35 @@ def test_merge_simulation_info(test_conf_file, test_tmp_folder): assert isfile(join(conf['simulator']['output'], 'merged_sim_data.dat')) del sim +@pytest.mark.test_tmp_folder +@pytest.mark.test_conf_file +@pytest.mark.parametrize('mode', ['simulator', 'mapper']) +def test_merge_data_info(test_conf_file, test_tmp_folder, mode): + conf = load_yaml_conf(test_conf_file) + conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} + pointing = get_point_source_info(conf['simulator']) + sim = RTACtoolsSimulation() + clock = 1 + for i in range(5): + sim.seed = i + datfile = join(conf['simulator']['output'], f'job_{i}_simulator.dat') + write_simulation_info(sim, conf['simulator'], pointing, datfile, clock) + assert isfile(datfile) + + datfile = join(conf['mapper']['output'], f'job_{i}_mapper.dat') + write_mapping_info(conf, datfile, clock) + assert isfile(datfile) + + log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log')) + merge_data_info(conf[mode], mode, log) + assert isfile(join(conf[mode]['output'], f'merged_{mode}_data.dat')) + del sim + @pytest.mark.test_conf_file def test_write_mapping_info(test_conf_file): conf = load_yaml_conf(test_conf_file) conf['simulator']['pointing'] = {'ra': 1, 'dec': 1} - datfile = join(conf['simulator']['output'], 'simulator.dat') + datfile = join(conf['mapper']['output'], 'mapper.dat') clock = 1 write_mapping_info(conf, datfile, clock) assert isfile(datfile) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 79ee504..7910354 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -141,6 +141,27 @@ def write_mapping_info(configuration, datfile, clock): with open(datfile, 'a') as f: f.write(f'{name} {seed} {exposure} {center_type} {pixelsize} {smooth} {clock}\n') +def merge_data_info(configuration, mode, log): + folder = configuration['output'] + datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f and mode in f] + merger = join(folder, f'merged_{mode}_data.dat') + # check merger file + if isfile(merger): + log.warning(f"Merger output already exists, overwrite {merger}") + f = open(merger, 'w+') + f.close() + # collect data + for i, datfile in enumerate(datfiles): + log.info(f"Collect data from {datfile}") + data = pd.read_csv(join(datfile), sep=' ') + if i == 0: + table = data + else: + table = pd.concat([table, data], ignore_index=True) + log.info(f"Lines in data: {len(table)}") + # write merger file + table.to_csv(merger, index=False, header=True, sep=' ', na_rep=np.nan) + def plot_map(fitsmap, log): plotmap = fitsmap.replace('.fits', '.png') plot = Plotter(log) -- GitLab From 3248f836b489dd57be30442969e1003ee43246c8 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 18 Oct 2023 11:43:38 +0200 Subject: [PATCH 94/98] fix merge_data_info --- astrort/utils/wrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 7910354..49f4dee 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -145,6 +145,7 @@ def merge_data_info(configuration, mode, log): folder = configuration['output'] datfiles = [join(folder, f) for f in listdir(folder) if '.dat' in f and 'job' in f and mode in f] merger = join(folder, f'merged_{mode}_data.dat') + log.info(f"Merger file: {merger}") # check merger file if isfile(merger): log.warning(f"Merger output already exists, overwrite {merger}") -- GitLab From bbed3fb588009bb0d5651c5d7ccde2829d28a678 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 18 Oct 2023 11:44:19 +0200 Subject: [PATCH 95/98] handy script to merge csv outputs --- astrort/scripts/merge_datfiles.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 astrort/scripts/merge_datfiles.py diff --git a/astrort/scripts/merge_datfiles.py b/astrort/scripts/merge_datfiles.py new file mode 100644 index 0000000..d44c490 --- /dev/null +++ b/astrort/scripts/merge_datfiles.py @@ -0,0 +1,30 @@ +# ***************************************************************************** +# Copyright (C) 2023 INAF +# This software is distributed under the terms of the BSD-3-Clause license +# +# Authors: +# Ambra Di Piano +# ***************************************************************************** + +import argparse +from os import listdir +from os.path import join, isfile +from astrort.utils.wrap import load_yaml_conf, merge_data_info +from astrort.configure.logging import set_logger + +def main(configuration, mode): + configuration = load_yaml_conf(configuration) + logfile = join(configuration[mode]['output'], f'merge_{mode}_data.log') + log = set_logger(configuration['logging']['level'], logfile) + log.info(f"merge {mode} data files") + merge_data_info(configuration[mode], mode, log) + log.info(f"{mode} merge completed") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file") + parser.add_argument('-m', '--mode', type=str, required=True, choices=['simulator', 'mapper'], help='Data table to merge') + args = parser.parse_args() + + main(args.configuration, args.mode) \ No newline at end of file -- GitLab From c1773f095d0e01a5ec1f0b324cb84a432bfadfaa Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 18 Oct 2023 12:29:13 +0200 Subject: [PATCH 96/98] remove unused imports --- astrort/scripts/merge_datfiles.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astrort/scripts/merge_datfiles.py b/astrort/scripts/merge_datfiles.py index d44c490..0f2a75e 100644 --- a/astrort/scripts/merge_datfiles.py +++ b/astrort/scripts/merge_datfiles.py @@ -7,8 +7,7 @@ # ***************************************************************************** import argparse -from os import listdir -from os.path import join, isfile +from os.path import join from astrort.utils.wrap import load_yaml_conf, merge_data_info from astrort.configure.logging import set_logger -- GitLab From 9e6a4630ffae0ea7a932d1b5e3fdab4e6bd9291d Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Wed, 18 Oct 2023 16:31:50 +0200 Subject: [PATCH 97/98] add fov to dat info --- astrort/utils/wrap.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index 49f4dee..ce8be41 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -97,15 +97,16 @@ def get_point_source_info(simulator): def write_simulation_info(simulator, configuration, pointing, datfile, clock): name = seeds_to_string_formatter(configuration['samples'], configuration['name'], configuration['seed']) seed = simulator.seed + fov = simulator.fov tstart, tstop = simulator.t duration = configuration['duration'] irf = configuration['irf'] point_ra, point_dec, offset, source_ra, source_dec = pointing['point_ra'], pointing['point_dec'], pointing['offset'], pointing['source_ra'], pointing['source_dec'] if not isfile(datfile): with open(datfile, 'w+') as f: - f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf computation_time\n') + f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf fov sim_time\n') with open(datfile, 'a') as f: - f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {irf} {clock}\n') + f.write(f'{name} {seed} {tstart} {tstop} {duration} {source_ra} {source_dec} {point_ra} {point_dec} {offset} {irf} {fov} {clock}\n') def merge_simulation_info(configuration, log): folder = configuration['output'] @@ -114,8 +115,8 @@ def merge_simulation_info(configuration, log): # check merger file if isfile(merger): log.warning(f"Merger output already exists, overwrite {merger}") - with open(merger, 'w+') as f: - f.write('name seed start stop duration source_ra source_dec point_ra point_dec offset irf computation_time\n') + f = open(merger, 'w+') + f.close() # collect data for i, datfile in enumerate(datfiles): log.info(f"Collect data from {datfile}") @@ -137,7 +138,7 @@ def write_mapping_info(configuration, datfile, clock): smooth = configuration['mapper']['smooth'] if not isfile(datfile): with open(datfile, 'w+') as f: - f.write('name seed exposure center_on pixelsize smooth computation_time\n') + f.write('name seed exposure center_on pixelsize smooth map_time\n') with open(datfile, 'a') as f: f.write(f'{name} {seed} {exposure} {center_type} {pixelsize} {smooth} {clock}\n') -- GitLab From fc1e0c9c16778ecdb49cab4eada24a5103f7fba5 Mon Sep 17 00:00:00 2001 From: Ambra Di Piano Date: Mon, 23 Oct 2023 16:13:41 +0200 Subject: [PATCH 98/98] fix simulation potingin --- astrort/utils/wrap.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astrort/utils/wrap.py b/astrort/utils/wrap.py index ce8be41..37876d6 100644 --- a/astrort/utils/wrap.py +++ b/astrort/utils/wrap.py @@ -58,8 +58,7 @@ def set_pointing(simulator, configuration, log): log.info(f"Randomising pointing coordinates [{point['point_ra']}, {point['point_dec']}]") else: point = get_point_source_info(configuration) - simulator.ra = point['point_ra'] - simulator.dec = point['point_dec'] + simulator.pointing = [point['point_ra'], point['point_dec']] return simulator, point def randomise_pointing_sim(simulator): -- GitLab