From 3b44dd964ac7eb5e7a010119dd9d8849c3eff9d0 Mon Sep 17 00:00:00 2001 From: acpaquette <acpaquette@usgs.gov> Date: Wed, 23 Sep 2020 10:27:43 -0700 Subject: [PATCH] Pvl 1.0.0 Update (#373) * Error throw changed from ParseError to LexemError * Fixed label comments to be pvl comments not python comments * Updated isis label parse to use the ISISGrammar and removed + from reserved characters * Fixed dawn and kaguya drivers * A small PVL fix and a bunch of datetime related fixes * Added pytz to the environment file * Removed strict argument from pvl.loads * Fixed another collections call --- ale/base/data_isis.py | 2 +- ale/base/label_isis.py | 9 +++--- ale/base/label_pds3.py | 2 +- ale/drivers/dawn_drivers.py | 31 ------------------- ale/drivers/lro_drivers.py | 4 +-- ale/drivers/ody_drivers.py | 6 ++-- ale/drivers/tgo_drivers.py | 2 +- ale/drivers/voyager_drivers.py | 2 +- environment.yml | 1 + .../JNCR_2016240_01M06152_V01_isis3.lbl | 4 +-- tests/pytests/test_isis_label.py | 8 ++--- tests/pytests/test_kaguya_drivers.py | 6 ++-- tests/pytests/test_pds3_label.py | 6 ++-- 13 files changed, 27 insertions(+), 56 deletions(-) diff --git a/ale/base/data_isis.py b/ale/base/data_isis.py index 905261e..3401c5e 100644 --- a/ale/base/data_isis.py +++ b/ale/base/data_isis.py @@ -79,7 +79,7 @@ def parse_table(table_label, data): offset += data_sizes[field['Type']] * field['Size'] # Parse the keywords from the label - results.update({key : value for key, value in table_label.items() if not isinstance(value, pvl._collections.PVLGroup)}) + results.update({key : value for key, value in table_label.items() if not isinstance(value, pvl.collections.PVLGroup)}) return results diff --git a/ale/base/label_isis.py b/ale/base/label_isis.py index e25e36d..66847e7 100644 --- a/ale/base/label_isis.py +++ b/ale/base/label_isis.py @@ -10,10 +10,11 @@ class IsisLabel(): if not hasattr(self, "_label"): if isinstance(self._file, pvl.PVLModule): self._label = self._file + grammar = pvl.grammar.ISISGrammar() try: - self._label = pvl.loads(self._file) + self._label = pvl.loads(self._file, grammar=grammar) except Exception: - self._label = pvl.load(self._file) + self._label = pvl.load(self._file, grammar=grammar) except: raise ValueError("{} is not a valid label".format(self._file)) return self._label @@ -236,7 +237,7 @@ class IsisLabel(): if 'ExposureDuration' in self.label['IsisCube']['Instrument']: exposure_duration = self.label['IsisCube']['Instrument']['ExposureDuration'] # Check for units on the PVL keyword - if isinstance(exposure_duration, pvl._collections.Units): + if isinstance(exposure_duration, pvl.collections.Quantity): units = exposure_duration.units if "ms" in units.lower() or 'milliseconds' in units.lower(): exposure_duration = exposure_duration.value * 0.001 @@ -261,7 +262,7 @@ class IsisLabel(): Line exposure duration in seconds """ line_exposure_duration = self.label['IsisCube']['Instrument']['LineExposureDuration'] - if isinstance(line_exposure_duration, pvl._collections.Units): + if isinstance(line_exposure_duration, pvl.collections.Quantity): units = line_exposure_duration.units if "ms" in units.lower(): line_exposure_duration = line_exposure_duration.value * 0.001 diff --git a/ale/base/label_pds3.py b/ale/base/label_pds3.py index fd5c995..ae15af0 100644 --- a/ale/base/label_pds3.py +++ b/ale/base/label_pds3.py @@ -12,7 +12,7 @@ class Pds3Label(): self._label = self._file else: try: - self._label = pvl.loads(self._file, strict=False) + self._label = pvl.loads(self._file) except Exception: self._label = pvl.load(self._file) except: diff --git a/ale/drivers/dawn_drivers.py b/ale/drivers/dawn_drivers.py index 163dd2f..2cfec88 100644 --- a/ale/drivers/dawn_drivers.py +++ b/ale/drivers/dawn_drivers.py @@ -39,37 +39,6 @@ class DawnFcPds3NaifSpiceDriver(Framer, Pds3Label, NaifSpice, Driver): return "{}_FILTER_{}".format(ID_LOOKUP[instrument_id], filter_number) - @property - def label(self): - """ - Loads a PVL from from the _file attribute and - parses the binary table data. - - Returns - ------- - PVLModule : - Dict-like object with PVL keys - """ - class PvlDecoder(pvl.decoder.PVLDecoder): - def unescape_next_char(self, stream): - esc = stream.read(1) - string = '\{}'.format(esc.decode('utf-8')).encode('utf-8') - return string - - if not hasattr(self, "_label"): - if isinstance(self._file, pvl.PVLModule): - self._label = self._file - try: - self._label = pvl.loads(self._file, PvlDecoder) - except Exception: - - # PvlDecoder class to ignore all escape sequences when getting - # the label - self._label = pvl.load(self._file, PvlDecoder) - except: - raise ValueError("{} is not a valid label".format(self._file)) - return self._label - @property def spacecraft_name(self): """ diff --git a/ale/drivers/lro_drivers.py b/ale/drivers/lro_drivers.py index 1c06695..00b6962 100644 --- a/ale/drivers/lro_drivers.py +++ b/ale/drivers/lro_drivers.py @@ -616,7 +616,7 @@ class LroMiniRfIsisLabelNaifSpiceDriver(Radar, NaifSpice, IsisLabel, Driver): : float start time """ - return spice.str2et(str(self.utc_start_time)) + return spice.str2et(self.utc_start_time.strftime("%Y-%m-%d %H:%M:%S.%f")) @property def ephemeris_stop_time(self): @@ -628,7 +628,7 @@ class LroMiniRfIsisLabelNaifSpiceDriver(Radar, NaifSpice, IsisLabel, Driver): : float stop time """ - return spice.str2et(str(self.utc_stop_time)) + return spice.str2et(self.utc_stop_time.strftime("%Y-%m-%d %H:%M:%S.%f")) @property def look_direction(self): diff --git a/ale/drivers/ody_drivers.py b/ale/drivers/ody_drivers.py index c55387c..05828af 100644 --- a/ale/drivers/ody_drivers.py +++ b/ale/drivers/ody_drivers.py @@ -50,7 +50,7 @@ class OdyThemisIrIsisLabelNaifSpiceDriver(LineScanner, IsisLabel, NaifSpice, NoD def ephemeris_start_time(self): og_start_time = super().ephemeris_start_time offset = self.label["IsisCube"]["Instrument"]["SpacecraftClockOffset"] - if isinstance(offset, pvl._collections.Units): + if isinstance(offset, pvl.collections.Quantity): units = offset.units if "ms" in units.lower(): offset = offset.value * 0.001 @@ -121,7 +121,7 @@ class OdyThemisVisIsisLabelNaifSpiceDriver(LineScanner, IsisLabel, NaifSpice, No og_start_time = super().ephemeris_start_time offset = self.label["IsisCube"]["Instrument"]["SpacecraftClockOffset"] - if isinstance(offset, pvl._collections.Units): + if isinstance(offset, pvl.collections.Quantity): units = offset.units if "ms" in units.lower(): offset = offset.value * 0.001 @@ -142,7 +142,7 @@ class OdyThemisVisIsisLabelNaifSpiceDriver(LineScanner, IsisLabel, NaifSpice, No Line exposure duration in seconds """ line_exposure_duration = self.label['IsisCube']['Instrument']['ExposureDuration'] - if isinstance(line_exposure_duration, pvl._collections.Units): + if isinstance(line_exposure_duration, pvl.collections.Quantity): units = line_exposure_duration.units if "ms" in units.lower(): line_exposure_duration = line_exposure_duration.value * 0.001 diff --git a/ale/drivers/tgo_drivers.py b/ale/drivers/tgo_drivers.py index 69b21f6..1f98023 100644 --- a/ale/drivers/tgo_drivers.py +++ b/ale/drivers/tgo_drivers.py @@ -50,7 +50,7 @@ class TGOCassisIsisLabelNaifSpiceDriver(Framer, IsisLabel, NaifSpice, NoDistorti : float ephemeris start time of the image. """ - return spice.utc2et(str(self.label['IsisCube']['Instrument']['StartTime'])) + return spice.utc2et(self.utc_start_time.strftime("%Y-%m-%d %H:%M:%S.%f")) @property def sensor_frame_id(self): diff --git a/ale/drivers/voyager_drivers.py b/ale/drivers/voyager_drivers.py index c1f9c26..e1d1743 100644 --- a/ale/drivers/voyager_drivers.py +++ b/ale/drivers/voyager_drivers.py @@ -49,7 +49,7 @@ class VoyagerCameraIsisLabelNaifSpiceDriver(Framer, IsisLabel, NaifSpice, Driver @property def ephemeris_start_time(self): - inital_time = spice.utc2et(self.utc_start_time.isoformat()) + inital_time = spice.utc2et(self.utc_start_time.strftime("%Y-%m-%d %H:%M:%S.%f")) # To get shutter end (close) time, subtract 2 seconds from the start time updated_time = inital_time - 2 # To get shutter start (open) time, take off the exposure duration from the end time. diff --git a/environment.yml b/environment.yml index 712eaef..db30c99 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: - nlohmann_json - numpy - pvl + - pytz - python - python-dateutil - scipy diff --git a/tests/pytests/data/JNCR_2016240_01M06152_V01/JNCR_2016240_01M06152_V01_isis3.lbl b/tests/pytests/data/JNCR_2016240_01M06152_V01/JNCR_2016240_01M06152_V01_isis3.lbl index 8874f31..339a2a3 100644 --- a/tests/pytests/data/JNCR_2016240_01M06152_V01/JNCR_2016240_01M06152_V01_isis3.lbl +++ b/tests/pytests/data/JNCR_2016240_01M06152_V01/JNCR_2016240_01M06152_V01_isis3.lbl @@ -26,10 +26,10 @@ Object = IsisCube InstrumentId = JNC TargetName = JUPITER - # Start time for the entire observation, i.e. start time for FrameNumber 1. + /* Start time for the entire observation, i.e. start time for FrameNumber 1. */ StartTime = 2016-08-27T09:00:04.129 - # Start count for the entire observation, i.e. start count for FrameNumber 1. + /* Start count for the entire observation, i.e. start count for FrameNumber 1. */ SpacecraftClockStartCount = 525560580:87 ExposureDuration = 204.800000 <ms> InterFrameDelay = 0.378 <s> diff --git a/tests/pytests/test_isis_label.py b/tests/pytests/test_isis_label.py index 8633e6d..ed05dbc 100644 --- a/tests/pytests/test_isis_label.py +++ b/tests/pytests/test_isis_label.py @@ -1,6 +1,6 @@ import pytest import pvl -from datetime import datetime +from datetime import datetime, timezone import ale from ale import base @@ -57,7 +57,7 @@ DpuId = DPU-A PivotAngle = -18.805847167969 <Degrees> Unlutted = 1 LutInversionTable = $messenger/calibration/LUT_INVERT/MDISLUTINV_0.TAB -# added to allow for testing +/* added to allow for testing */ LineExposureDuration = 1000 End_Group @@ -113,10 +113,10 @@ def test_spacecraft_clock_stop_count(test_cube_label): assert test_cube_label.spacecraft_clock_stop_count == "1/0089576657:990000" def test_utc_start_time(test_cube_label): - assert test_cube_label.utc_start_time == datetime(2007, 6, 6, 00, 22, 10, 751814) + assert test_cube_label.utc_start_time == datetime(2007, 6, 6, 00, 22, 10, 751814, timezone.utc) def test_utc_stop_time(test_cube_label): - assert test_cube_label.utc_stop_time == datetime(2007, 6, 6, 00, 22, 10, 768814) + assert test_cube_label.utc_stop_time == datetime(2007, 6, 6, 00, 22, 10, 768814, timezone.utc) def test_target_name(test_cube_label): assert test_cube_label.target_name.lower() == "venus" diff --git a/tests/pytests/test_kaguya_drivers.py b/tests/pytests/test_kaguya_drivers.py index 6b6b423..3b8b70b 100644 --- a/tests/pytests/test_kaguya_drivers.py +++ b/tests/pytests/test_kaguya_drivers.py @@ -1,7 +1,7 @@ import pytest import os import numpy as np -import datetime +from datetime import datetime, timezone import spiceypy as spice from importlib import reload import json @@ -56,10 +56,10 @@ class test_pds_naif(unittest.TestCase): assert self.driver.short_mission_name == 'selene' def test_utc_start_time(self): - assert self.driver.utc_start_time == datetime.datetime(2009, 4, 5, 20, 9, 53, 607478, tzinfo=datetime.timezone.utc) + assert self.driver.utc_start_time == datetime(2009, 4, 5, 20, 9, 53, 607478, timezone.utc) def test_utc_stop_time(self): - assert self.driver.utc_stop_time == datetime.datetime(2009, 4, 5, 20, 10, 23, 864978, tzinfo=datetime.timezone.utc) + assert self.driver.utc_stop_time == datetime(2009, 4, 5, 20, 10, 23, 864978, timezone.utc) def test_instrument_id(self): assert self.driver.instrument_id == 'LISM_TC1_STF' diff --git a/tests/pytests/test_pds3_label.py b/tests/pytests/test_pds3_label.py index 9446503..5ea655c 100644 --- a/tests/pytests/test_pds3_label.py +++ b/tests/pytests/test_pds3_label.py @@ -1,7 +1,7 @@ import pytest import pvl -from datetime import datetime +from datetime import datetime, timezone import ale from ale import base from ale.base.label_pds3 import Pds3Label @@ -84,10 +84,10 @@ def test_instrument_host_name(test_image_label): assert test_image_label.instrument_host_name.lower() == 'mars reconnaissance orbiter' def test_utc_start_time(test_image_label): - assert test_image_label.utc_start_time == datetime(2006, 11, 1, 22, 45, 53, 570000) + assert test_image_label.utc_start_time == datetime(2006, 11, 1, 22, 45, 53, 570000, timezone.utc) def test_utc_stop_time(test_image_label): - assert test_image_label.utc_stop_time == datetime(2006, 11, 1, 23, 49, 50, 370000) + assert test_image_label.utc_stop_time == datetime(2006, 11, 1, 23, 49, 50, 370000, timezone.utc) def test_image_lines(test_image_label): assert test_image_label.image_lines == 2432 -- GitLab