From baff4431ffdba179097e3c1fbae78c0437819045 Mon Sep 17 00:00:00 2001 From: acpaquette Date: Tue, 16 Jul 2019 11:05:06 -0700 Subject: [PATCH] General ISD Generation Fixes + Dawn Notebook (#206) * Small import fixes to dawn * More imports and updated the start_time to utc_start_time * Added properties to dawn driver to get things working * General changes to bring the isd back in line with isds generated in late April * Update to dawn driver for detector sample and line center * Added dawn notebook * Fixed small type in sensor * Fixed load/loads function * Removed unnecessary print from load function * Updated formatter test * Updated load and loads to use a default formatter * Minor import shuffling * Reverted center ephemeris time * Updated camera driver mixins to set the ephemeris stop time * Suggested changes and test updates * Removed weird driver importing * Updated return in data_naif * Small changes to get tests passing * Updated line scan center ephemeris doc string * Removed stop time from tests as it is computed from other properties --- ale/base/data_naif.py | 2 +- ale/base/label_pds3.py | 2 +- ale/base/type_sensor.py | 35 +++- ale/drivers/__init__.py | 38 ++-- ale/drivers/dawn_drivers.py | 14 +- ale/drivers/mro_drivers.py | 1 - ale/formatters/usgscsm_formatter.py | 6 +- .../write_DawnFcPds3NaifSpiceDriver.ipynb | 170 ++++++++++++++++++ tests/pytests/test_usgscsm_formatter.py | 17 +- 9 files changed, 248 insertions(+), 37 deletions(-) create mode 100644 notebooks/write_DawnFcPds3NaifSpiceDriver.ipynb diff --git a/ale/base/data_naif.py b/ale/base/data_naif.py index b811f33..cbcb11a 100644 --- a/ale/base/data_naif.py +++ b/ale/base/data_naif.py @@ -431,7 +431,7 @@ class NaifSpice(): : double Center ephemeris time for an image """ - return (self.ephemeris_start_time + self.ephemeris_stop_time)/2 + return (self.ephemeris_start_time + self.ephemeris_stop_time) / 2 @property def detector_center_sample(self): diff --git a/ale/base/label_pds3.py b/ale/base/label_pds3.py index fb204ff..16f743d 100644 --- a/ale/base/label_pds3.py +++ b/ale/base/label_pds3.py @@ -287,7 +287,7 @@ class Pds3Label(): try: unit = self.label['EXPOSURE_DURATION'].units unit = unit.lower() - if unit == "ms" or unit == "msec": + if unit == "ms" or unit == "msec" or unit == "millisecond": return self.label['EXPOSURE_DURATION'].value * 0.001 else: return self.label['EXPOSURE_DURATION'].value diff --git a/ale/base/type_sensor.py b/ale/base/type_sensor.py index 88e58a3..f2d427e 100644 --- a/ale/base/type_sensor.py +++ b/ale/base/type_sensor.py @@ -49,6 +49,22 @@ class LineScanner(): """ return np.linspace(self.ephemeris_start_time, self.ephemeris_stop_time, self.image_lines / 64) + @property + def ephemeris_stop_time(self): + """ + Returns the sum of the starting ephemeris time and the number of lines + times the exposure duration. Expects ephemeris start time, exposure duration + and image lines to be defined. These should be double precision numbers + containing the ephemeris start, exposure duration and number of lines of + the image. + + Returns + ------- + : double + Center ephemeris time for an image + """ + return self.ephemeris_start_time + (self.image_lines * self.exposure_duration) + class Framer(): @property def name_model(self): @@ -66,8 +82,8 @@ class Framer(): @property def ephemeris_time(self): """ - Returns the center ephemeris time for the image which is the average - of the start and stop ephemeris time. + Returns the center ephemeris time for the image which is start time plus + half of the exposure duration. Expects center_ephemeris_time to be defined. This should be a double containing the average of the start and stop ephemeris times. @@ -77,3 +93,18 @@ class Framer(): Center ephemeris time for the image """ return [self.center_ephemeris_time] + + @property + def ephemeris_stop_time(self): + """ + Returns the sum of the starting ephemeris time and the exposure duration. + Expects ephemeris start time and exposure duration to be defined. These + should be double precision numbers containing the ephemeris start and + exposure duration of the image. + + Returns + ------- + : double + Ephemeris stop time for an image + """ + return self.ephemeris_start_time + self.exposure_duration diff --git a/ale/drivers/__init__.py b/ale/drivers/__init__.py index 76f2bd4..9bfaeaa 100644 --- a/ale/drivers/__init__.py +++ b/ale/drivers/__init__.py @@ -9,18 +9,23 @@ import os from glob import glob import json import numpy as np +import datetime from datetime import datetime, date +import traceback -from abc import ABC +from ale.formatters.usgscsm_formatter import to_usgscsm +from ale.formatters.isis_formatter import to_isis -import datetime +from abc import ABC # dynamically load drivers __all__ = [os.path.splitext(os.path.basename(d))[0] for d in glob(os.path.join(os.path.dirname(__file__), '*_drivers.py'))] __driver_modules__ = [importlib.import_module('.'+m, package='ale.drivers') for m in __all__] -drivers = dict(chain.from_iterable(inspect.getmembers(dmod, lambda x: inspect.isclass(x) and "_driver" in x.__module__) for dmod in __driver_modules__)) +__formatters__ = {'usgscsm': to_usgscsm, + 'isis': to_isis} +drivers = dict(chain.from_iterable(inspect.getmembers(dmod, lambda x: inspect.isclass(x) and "_driver" in x.__module__) for dmod in __driver_modules__)) class JsonEncoder(json.JSONEncoder): def default(self, obj): @@ -41,7 +46,7 @@ class JsonEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) -def load(label): +def load(label, formatter='usgscsm'): """ Attempt to load a given label from all possible drivers @@ -50,20 +55,21 @@ def load(label): label : str String path to the given label file """ + if isinstance(formatter, str): + formatter = __formatters__[formatter] + for name, driver in drivers.items(): - print("Trying:", name) - try: - res = driver(label) - if res.is_valid(): - with res as r: - return res.to_dict() - except Exception as e: - import traceback - print("Driver Failed:", e) - traceback.print_exc() + print(f'Trying {name}') + try: + res = driver(label) + with res as driver: + return formatter(driver) + except Exception as e: + print(f'Failed: {e}\n') + traceback.print_exc() raise Exception('No Such Driver for Label') -def loads(label): - res = load(label) +def loads(label, formatter='usgscsm'): + res = load(label, formatter) return json.dumps(res, cls=JsonEncoder) diff --git a/ale/drivers/dawn_drivers.py b/ale/drivers/dawn_drivers.py index f637731..f0ea7bf 100644 --- a/ale/drivers/dawn_drivers.py +++ b/ale/drivers/dawn_drivers.py @@ -127,11 +127,11 @@ class DawnFcPds3NaifSpiceDriver(Pds3Label, NaifSpice, Framer, Driver): via a spice call but 193 ms needs to be added to account for the CCD being discharged or cleared. """ - if not hasattr(self, '_starting_ephemeris_time'): + if not hasattr(self, '_ephemeris_start_time'): sclock = self.spacecraft_clock_start_count - self._starting_ephemeris_time = spice.scs2e(self.spacecraft_id, sclock) - self._starting_ephemeris_time += 193.0 / 1000.0 - return self._starting_ephemeris_time + self._ephemeris_start_time = spice.scs2e(self.spacecraft_id, sclock) + self._ephemeris_start_time += 193.0 / 1000.0 + return self._ephemeris_start_time @property def usgscsm_distortion_model(self): @@ -207,7 +207,7 @@ class DawnFcPds3NaifSpiceDriver(Pds3Label, NaifSpice, Framer, Driver): Detector line corresponding to the first image line """ return 1 - + @property def detector_start_sample(self): """ @@ -241,7 +241,7 @@ class DawnFcPds3NaifSpiceDriver(Pds3Label, NaifSpice, Framer, Driver): : float center detector sample """ - return float(spice.gdpool('INS{}_BORESIGHT'.format(self.ikid), 0, 3)[0]) + return float(spice.gdpool('INS{}_CCD_CENTER'.format(self.ikid), 0, 2)[0]) @property def detector_center_line(self): @@ -254,4 +254,4 @@ class DawnFcPds3NaifSpiceDriver(Pds3Label, NaifSpice, Framer, Driver): : float center detector line """ - return float(spice.gdpool('INS{}_BORESIGHT'.format(self.ikid), 0, 3)[1]) + return float(spice.gdpool('INS{}_CCD_CENTER'.format(self.ikid), 0, 2)[1]) diff --git a/ale/drivers/mro_drivers.py b/ale/drivers/mro_drivers.py index 2f62fb1..48c4b86 100644 --- a/ale/drivers/mro_drivers.py +++ b/ale/drivers/mro_drivers.py @@ -61,7 +61,6 @@ class MroCtxIsisLabelIsisSpiceDriver(Driver, IsisSpice, LineScanner, RadialDisto """ return self.label["IsisCube"]["Instrument"]["LineExposureDuration"].value * 0.001 # Scale to seconds - class MroCtxIsisLabelNaifSpiceDriver(IsisLabel, NaifSpice, LineScanner, RadialDistortion, Driver): """ Driver for reading CTX ISIS labels. diff --git a/ale/formatters/usgscsm_formatter.py b/ale/formatters/usgscsm_formatter.py index 703a61e..d56d9d9 100644 --- a/ale/formatters/usgscsm_formatter.py +++ b/ale/formatters/usgscsm_formatter.py @@ -23,7 +23,7 @@ def to_usgscsm(driver): body_radii = driver.target_body_radii isd_data['radii'] = { 'semimajor' : body_radii[0], - 'semiminor' : body_radii[1], + 'semiminor' : body_radii[2], 'unit' : 'km' } positions, velocities, position_times = driver.sensor_position @@ -81,7 +81,7 @@ def to_usgscsm(driver): isd_data['name_model'] = 'USGS_ASTRO_LINE_SCANNER_SENSOR_MODEL' isd_data['interpolation_method'] = 'lagrange' start_lines, start_times, scan_rates = driver.line_scan_rate - center_time = (driver.ephemeris_stop_time + driver.ephemeris_start_time) / 2 + center_time = driver.center_ephemeris_time isd_data['line_scan_rate'] = [[line, time, rate] for line, time, rate in zip(start_lines, start_times, scan_rates)] isd_data['starting_ephemeris_time'] = driver.ephemeris_start_time isd_data['center_ephemeris_time'] = center_time @@ -100,7 +100,7 @@ def to_usgscsm(driver): # frame sensor model specifics if isinstance(driver, Framer): isd_data['name_model'] = 'USGS_ASTRO_FRAME_SENSOR_MODEL' - isd_data['center_ephemeris_time'] = position_times[0] + isd_data['center_ephemeris_time'] = driver.center_ephemeris_time # check that there is a valid sensor model name if 'name_model' not in isd_data: diff --git a/notebooks/write_DawnFcPds3NaifSpiceDriver.ipynb b/notebooks/write_DawnFcPds3NaifSpiceDriver.ipynb new file mode 100644 index 0000000..4d99686 --- /dev/null +++ b/notebooks/write_DawnFcPds3NaifSpiceDriver.ipynb @@ -0,0 +1,170 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Writing out a USGSCSM ISD from a PDS3 Dawn Framing Camera image" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "import ale\n", + "from ale.drivers.dawn_drivers import DawnFcPds3NaifSpiceDriver\n", + "from ale.formatters.usgscsm_formatter import to_usgscsm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiating an ALE driver\n", + "\n", + "ALE drivers are objects that define how to acquire common ISD keys from an input image format, in this case we are reading in a PDS3 image using NAIF SPICE kernels for exterior orientation data. If the driver utilizes NAIF SPICE kernels, it is implemented as a [context manager](https://docs.python.org/3/reference/datamodel.html#context-managers) and will furnish metakernels when entering the context (i.e. when entering the `with` block) and free the metakernels on exit. This maintains the integrity of spicelib's internal data structures. These driver objects are short-lived and are input to a formatter function that consumes the API to create a serializable file format. `ale.formatters` contains available formatter functions. \n", + "\n", + "The default config file is located at `ale/config.yml` and is copied into your home directory at `.ale/config.yml` on first use of the library. The config file can be modified using a text editor. `ale.config` is loaded into memory as a dictionary. It is used to find metakernels for different missions. For example, there is an entry for Dawn that points to `/usgs/cpkgs/isis3/data/dawn/kernels/mk/` by default. If you want to use your own metakernels, you will need to udpate this path. For example, if the metakernels are located in `/data/dawn/mk/` the Dawn entry should be updated with this path. If you are using the default metakernels, then you do not need to update the path.\n", + "\n", + "ALE has a two step process for writing out an ISD: 1. Instantiate your driver (in this case `DawnFcPds3LabelNaifSpiceDriver`) within a context and 2. pass the driver object into a formatter (in this case, `to_usgscsm`). \n", + "\n", + "Requirements:\n", + " * A PDS3 Dawn image\n", + " * NAIF metakernels installed\n", + " * Config file path for Dawn (ale.config.dawn) pointing to Dawn NAIF metakernel directory \n", + " * A conda environment with ALE installed into it usisng the `conda install` command or created using the environment.yml file at the base of ALE." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cassini: /usgs/cpkgs/isis3/data/cassini/kernels/mk/\n", + "dawn: /data/spice/dawn-m_a-spice-6-v1.0/dawnsp_1000/extras/mk\n", + "kaguya: /data/spice/SELENE/kernels/mk/\n", + "lro: /scratch/jlaura/spice/lro-l-spice-6-v1.0/lrosp_1000/extras/mk/\n", + "mdis: /data/spice/mess-e_v_h-spice-6-v1.0/messsp_1000/extras/mk\n", + "mro: /data/spice/mro-m-spice-6-v1.0/mrosp_1000/extras/mk\n", + "spice_root: /data/spice/\n", + "\n", + "Dawn spice directory: /data/spice/dawn-m_a-spice-6-v1.0/dawnsp_1000/extras/mk\n" + ] + } + ], + "source": [ + "# printing config displays the yaml formatted string\n", + "print(ale.config)\n", + "\n", + "# config object is a dictionary so it has the same access patterns\n", + "print('Dawn spice directory:', ale.config['dawn'])\n", + "\n", + "# updating config for new Dawn path in this notebook \n", + "# Note: this will not change the path in `.ale/config.yml`. This change only lives in the notebook.\n", + "# ale.config['dawn'] = '/scratch/jlaura/spice/dawn-m_a-spice-6-v1.0/dawnsp_1000/extras/mk/'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# change to desired PDS3 image path\n", + "file_name = 'FC21A0039691_15231053805F1E.IMG'\n", + "\n", + "# metakernels are furnsh-ed when entering the context (with block) with a driver instance\n", + "# most driver constructors simply accept an image path \n", + "with DawnFcPds3NaifSpiceDriver(file_name) as driver:\n", + " # pass driver instance into formatter function\n", + " usgscsmString = to_usgscsm(driver)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write ISD to disk \n", + "\n", + "ALE formatter functions generally return bytes or a string that can be written out to disk. ALE's USGSCSM formatter function returns a JSON encoded string that can be written out using any JSON library. \n", + "\n", + "USGSCSM requires the ISD to be colocated with the image file with a `.json` extension in place of the image extension." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['radii', 'sensor_position', 'sun_position', 'sensor_orientation', 'detector_sample_summing', 'detector_line_summing', 'focal_length_model', 'detector_center', 'starting_detector_line', 'starting_detector_sample', 'focal2pixel_lines', 'focal2pixel_samples', 'optical_distortion', 'image_lines', 'image_samples', 'name_platform', 'name_sensor', 'reference_height', 'name_model', 'center_ephemeris_time'])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Load the json string into a dict\n", + "usgscsm_dict = json.loads(usgscsmString)\n", + "\n", + "# Write the dict out to the associated file\n", + "json_file = os.path.splitext(file_name)[0] + '.json'\n", + "\n", + "# Save off the json and read it back in to check if\n", + "# the json exists and was formatted correctly\n", + "with open(json_file, 'w') as fp:\n", + " json.dump(usgscsm_dict, fp)\n", + " \n", + "with open(json_file, 'r') as fp:\n", + " usgscsm_dict = json.load(fp)\n", + " \n", + "usgscsm_dict.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ale_env", + "language": "python", + "name": "ale_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/pytests/test_usgscsm_formatter.py b/tests/pytests/test_usgscsm_formatter.py index 7f1332e..c9e1cf8 100644 --- a/tests/pytests/test_usgscsm_formatter.py +++ b/tests/pytests/test_usgscsm_formatter.py @@ -6,15 +6,16 @@ from ale.formatters import usgscsm_formatter from ale.base.base import Driver from ale.base.type_sensor import LineScanner, Framer from ale.transformation import FrameNode +from ale.base.data_naif import NaifSpice from ale.rotation import ConstantRotation, TimeDependentRotation -class TestDriver(Driver): +class TestDriver(Driver, NaifSpice): """ Test Driver implementation with dummy values """ @property def target_body_radii(self): - return (1100, 1000) + return (1100, 1100, 1000) @property def frame_chain(self): @@ -73,14 +74,14 @@ class TestDriver(Driver): def platform_name(self): return 'Test Platform' - @property - def ephemeris_stop_time(self): - return 900 - @property def ephemeris_start_time(self): return 800 + @property + def exposure_duration(self): + return 100 + @property def focal2pixel_lines(self): return [0.1, 0.2, 0.3] @@ -161,6 +162,10 @@ class TestLineScanner(LineScanner, TestDriver): def image_lines(self): return 10000 + @property + def exposure_duration(self): + return .01 + class TestFramer(Framer, TestDriver): """ -- GitLab