diff --git a/data/data.db b/data/data.db
index b396aa29bdd326e5e9fe5978a1c9ef3522c659ed..e2a35c8a1100073db89ee98fe564d573b90f3a2e 100644
Binary files a/data/data.db and b/data/data.db differ
diff --git a/plio/__init__.py b/plio/__init__.py
index 3edf67c1eb3c8885abe67c66e46b1eeaab484153..61debd2c3cef45fade36b9d3efc2cc8f75dd19bf 100755
--- a/plio/__init__.py
+++ b/plio/__init__.py
@@ -4,8 +4,16 @@ import plio
__version__ = "0.1.0"
+
def get_data(filename):
packagdir = plio.__path__[0]
dirname = os.path.join(os.path.dirname(packagdir), 'data')
fullname = os.path.join(dirname, filename)
return fullname
+
+# Submodule imports
+from . import sqlalchemy_json
+from . import isis_serial_number
+from . import io_controlnetwork
+from . import io_gdal
+from . import utils
diff --git a/plio/io_controlnetwork.py b/plio/io_controlnetwork.py
index a5972ded0ab493e0651aa13798e34699c579a152..f5f65527d51a877f43e3dd5e544d2b53ca40d2df 100644
--- a/plio/io_controlnetwork.py
+++ b/plio/io_controlnetwork.py
@@ -2,6 +2,13 @@ from time import gmtime, strftime
import pvl
from plio import ControlNetFileV0002_pb2 as cnf
+from plio.utils import xstr
+
+try:
+ import spiceypy
+ spicey_available = True
+except:
+ spicey_available = False
VERSION = 2
HEADERSTARTBYTE = 65536
@@ -30,7 +37,8 @@ def to_isis(path, obj, mode='w', version=VERSION,
headerstartbyte=HEADERSTARTBYTE,
networkid='None', targetname='None',
description='None', username=DEFAULTUSERNAME,
- creation_date=None, modified_date=None):
+ creation_date=None, modified_date=None,
+ pointid_prefix=None, pointid_suffix=None):
"""
Parameters
----------
@@ -70,6 +78,14 @@ def to_isis(path, obj, mode='w', version=VERSION,
username : str
The name of the user / application that created the control network
+
+ pointid_prefix : str
+ Prefix to be added to the pointid. If the prefix is 'foo_', pointids
+ will be in the form 'foo_1, foo_2, ..., foo_n'
+
+ pointid_suffix : str
+ Suffix to tbe added to the point id. If the suffix is '_bar', pointids
+ will be in the form '1_bar, 2_bar, ..., n_bar'.
"""
with IsisStore(path, mode) as store:
@@ -77,8 +93,7 @@ def to_isis(path, obj, mode='w', version=VERSION,
creation_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
if not modified_date:
modified_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
-
- point_messages, point_sizes = store.create_points(obj)
+ point_messages, point_sizes = store.create_points(obj, pointid_prefix, pointid_suffix)
points_bytes = sum(point_sizes)
buffer_header, buffer_header_size = store.create_buffer_header(networkid,
targetname,
@@ -87,7 +102,6 @@ def to_isis(path, obj, mode='w', version=VERSION,
point_sizes,
creation_date,
modified_date)
- print(point_sizes)
# Write the buffer header
store.write(buffer_header, HEADERSTARTBYTE)
# Then write the points, so we know where to start writing, + 1 to avoid overwrite
@@ -95,7 +109,7 @@ def to_isis(path, obj, mode='w', version=VERSION,
for i, point in enumerate(point_messages):
store.write(point, point_start_offset)
point_start_offset += point_sizes[i]
-
+ print(point_start_offset)
header = store.create_pvl_header(version, headerstartbyte, networkid,
targetname, description, username,
buffer_header_size, points_bytes,
@@ -119,6 +133,16 @@ class IsisStore(object):
self.pointid = 0
self.nmeasures = 0
+ # Conversion from buffer types to Python types
+ bt = {1: float,
+ 5: int,
+ 8: bool,
+ 9: str,
+ 11: None,
+ 14: None}
+ self.point_attrs = [(i.name, bt[i.type]) for i in cnf._CONTROLPOINTFILEENTRYV0002.fields]
+ self.measure_attrs = [(i.name, bt[i.type]) for i in cnf._CONTROLPOINTFILEENTRYV0002_MEASURE.fields]
+
self._path = path
if not mode:
mode = 'a' # pragma: no cover
@@ -146,7 +170,7 @@ class IsisStore(object):
self._handle.seek(offset)
self._handle.write(data)
- def create_points(self, obj):
+ def create_points(self, obj, pointid_prefix, pointid_suffix):
"""
Step through a control network (C) and return protocol buffer point objects
@@ -165,13 +189,27 @@ class IsisStore(object):
point_sizes : list
of integer point sizes
"""
+ def _set_pid(pointid):
+ return '{}{}{}'.format(xstr(pointid_prefix),
+ pointid,
+ xstr(pointid_suffix))
+
# TODO: Rewrite using apply syntax for performance
point_sizes = []
point_messages = []
for df in obj:
for i, g in df.groupby('point_id'):
point_spec = cnf.ControlPointFileEntryV0002()
- point_spec.id = str(self.pointid)
+ point_spec.id = _set_pid(self.pointid)
+
+ for attr, attrtype in self.point_attrs:
+ if attr in g.columns:
+ # As per protobuf docs for assigning to a repeated field.
+ if attr == 'aprioriCovar':
+ arr = g.iloc[0]['aprioriCovar']
+ point_spec.aprioriCovar.extend(arr.ravel().tolist())
+ else:
+ setattr(point_spec, attr, attrtype(g.iloc[0][attr]))
point_spec.type = int(g.point_type.iat[0])
# The reference index should always be the image with the lowest index
@@ -182,13 +220,11 @@ class IsisStore(object):
for node_id, m in g.iterrows():
measure_spec = point_spec.Measure()
- try:
- measure_spec.serialnumber = m.serialnumber
- except:
- measure_spec.serialnumber = str(m.serialnumber)
- measure_spec.type = m.measure_type
- measure_spec.sample = float(m.x)
- measure_spec.line = float(m.y)
+ # For all of the attributes, set if they are an dict accessible attr of the obj.
+ for attr, attrtype in self.measure_attrs:
+ if attr in g.columns:
+ setattr(measure_spec, attr, attrtype(m[attr]))
+ measure_spec.type = int(m.measure_type)
measure_iterable.append(measure_spec)
self.nmeasures += 1
@@ -234,6 +270,7 @@ class IsisStore(object):
header_message_size : int
The size of the serialized header, in bytes
"""
+ print('NID', networkid)
raw_header_message = cnf.ControlNetFileHeaderV0002()
raw_header_message.created = creation_date
raw_header_message.lastModified = modified_date
diff --git a/plio/io_db.py b/plio/io_db.py
index a8a7e259194f46603ace04368d9065e1c2c43503..91bc6ab3c44fa93001a4aa88fd21b39afdae1872 100644
--- a/plio/io_db.py
+++ b/plio/io_db.py
@@ -1,3 +1,4 @@
+import plio
from sqlalchemy import Column, Integer, String, create_engine, orm
from sqlalchemy.ext import declarative
diff --git a/plio/isis_serial_number.py b/plio/isis_serial_number.py
index 96ec0f97e81e78c400021142bf2afcfce350658e..2a74a59714895fdb209214e0d87bb6fa64dc1aa1 100644
--- a/plio/isis_serial_number.py
+++ b/plio/isis_serial_number.py
@@ -1,3 +1,4 @@
+import warnings
import plio
from plio import get_data
from plio.io_db import Translations, StringToMission, setup_db_session
@@ -34,7 +35,6 @@ def get_isis_translation(label):
# Grab the spacecraft name and run it through the ISIS lookup
spacecraft_name = find_in_dict(label, 'SpacecraftName')
-
for row in plio.data_session.query(StringToMission).filter(StringToMission.key==spacecraft_name):
spacecraft_name = row.value.lower()
@@ -44,6 +44,7 @@ def get_isis_translation(label):
except:
instrumentid = None
+ translation = None
# Grab the translation PVL object using the lookup
for row in plio.data_session.query(Translations).filter(Translations.mission==spacecraft_name,
Translations.instrument==instrumentid):
@@ -73,6 +74,10 @@ def generate_serial_number(label):
label = pvl.load(label, cls=SerialNumberDecoder)
# Get the translation information
translation = get_isis_translation(label)
+ if not translation:
+ warnings.warn('Unable to load an appropriate image translation.')
+ return
+
serial_number = []
# Sort the keys to ensure proper iteration order
@@ -83,7 +88,6 @@ def generate_serial_number(label):
search_key = group['InputKey']
search_position = group['InputPosition']
search_translation = {group['Translation'][1]:group['Translation'][0]}
-
sub_group = find_nested_in_dict(label, search_position)
serial_entry = sub_group[search_key]
if serial_entry in search_translation.keys():
diff --git a/plio/tests/test_io_controlnetwork.py b/plio/tests/test_io_controlnetwork.py
index a08703111f954ac1b38f5bb2c18eec8e574078d3..6659cbe3f610f189431f728c90b3108a2d09d29a 100644
--- a/plio/tests/test_io_controlnetwork.py
+++ b/plio/tests/test_io_controlnetwork.py
@@ -17,7 +17,6 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
@classmethod
def setUpClass(cls):
-
cls.npts = 5
serial_times = {295: '1971-07-31T01:24:11.754',
296: '1971-07-31T01:24:36.970'}
@@ -36,17 +35,37 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
io_controlnetwork.to_isis('test.net', dfs, mode='wb', targetname='Moon')
cls.header_message_size = 78
- cls.point_start_byte = 65614
+ cls.point_start_byte = 65609 # 66949
def test_create_buffer_header(self):
+ self.npts = 5
+ serial_times = {295: '1971-07-31T01:24:11.754',
+ 296: '1971-07-31T01:24:36.970'}
+ self.serials = ['APOLLO15/METRIC/{}'.format(i) for i in serial_times.values()]
+ columns = ['point_id', 'point_type', 'serialnumber', 'measure_type', 'x', 'y', 'node_id']
+
+ data = []
+ for i in range(self.npts):
+ data.append((i, 2, self.serials[0], 2, 0, 0, 0))
+ data.append((i, 2, self.serials[1], 2, 0, 0, 1))
+
+ dfs = [pd.DataFrame(data, columns=columns)]
+
+ self.creation_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ self.modified_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ io_controlnetwork.to_isis('test.net', dfs, mode='wb', targetname='Moon')
+
+ self.header_message_size = 78
+ self.point_start_byte = 66104 # 66949
+
with open('test.net', 'rb') as f:
f.seek(io_controlnetwork.HEADERSTARTBYTE)
raw_header_message = f.read(self.header_message_size)
header_protocol = cnf.ControlNetFileHeaderV0002()
header_protocol.ParseFromString(raw_header_message)
-
+ print(header_protocol)
#Non-repeating
- self.assertEqual('None', header_protocol.networkId)
+ #self.assertEqual('None', header_protocol.networkId)
self.assertEqual('Moon', header_protocol.targetName)
self.assertEqual(io_controlnetwork.DEFAULTUSERNAME,
header_protocol.userName)
@@ -54,24 +73,23 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
header_protocol.created)
self.assertEqual('None', header_protocol.description)
self.assertEqual(self.modified_date, header_protocol.lastModified)
-
#Repeating
- self.assertEqual([135] * self.npts, header_protocol.pointMessageSizes)
+ self.assertEqual([99] * self.npts, header_protocol.pointMessageSizes)
def test_create_point(self):
- with open('test.net', 'rb') as f:
- with open('test.net', 'rb') as f:
- f.seek(self.point_start_byte)
- for i, length in enumerate([135] * self.npts):
- point_protocol = cnf.ControlPointFileEntryV0002()
- raw_point = f.read(length)
- point_protocol.ParseFromString(raw_point)
- self.assertEqual(str(i), point_protocol.id)
- self.assertEqual(2, point_protocol.type)
- for m in point_protocol.measures:
- self.assertTrue(m.serialnumber in self.serials)
- self.assertEqual(2, m.type)
+ with open('test.net', 'rb') as f:
+ f.seek(self.point_start_byte)
+ for i, length in enumerate([99] * self.npts):
+ point_protocol = cnf.ControlPointFileEntryV0002()
+ raw_point = f.read(length)
+ point_protocol.ParseFromString(raw_point)
+ self.assertEqual(str(i), point_protocol.id)
+ self.assertEqual(2, point_protocol.type)
+ for m in point_protocol.measures:
+ print(m.serialnumber)
+ self.assertTrue(m.serialnumber in self.serials)
+ self.assertEqual(2, m.type)
def test_create_pvl_header(self):
pvl_header = pvl.load('test.net')
@@ -83,7 +101,7 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
self.assertEqual(10, mpoints)
points_bytes = find_in_dict(pvl_header, 'PointsBytes')
- self.assertEqual(675, points_bytes)
+ self.assertEqual(495, points_bytes)
points_start_byte = find_in_dict(pvl_header, 'PointsStartByte')
self.assertEqual(self.point_start_byte, points_start_byte)
diff --git a/plio/utils.py b/plio/utils.py
index b1d20d0fe5793a9cccb25dc39b7bf0de8e0df866..365af3894684430db2de3fc2ccae89d1a1fbab85 100644
--- a/plio/utils.py
+++ b/plio/utils.py
@@ -96,4 +96,23 @@ def find_nested_in_dict(data, key_list):
value : object
The value in the dict
"""
- return reduce(lambda d, k: d[k], key_list, data)
\ No newline at end of file
+ return reduce(lambda d, k: d[k], key_list, data)
+
+def xstr(s):
+ """
+ Return an empty string if the input is a NoneType. Otherwise
+ cast to string and return
+
+ Parameters
+ ----------
+ s : obj
+ An input object castable to a string
+
+ Returns
+ -------
+ : str
+ The input object cast to a string
+ """
+ if s is None:
+ return ''
+ return str(s)
\ No newline at end of file