Skip to content
Snippets Groups Projects
Commit e0c3fc93 authored by jay's avatar jay
Browse files

Updates

parent acbe121f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
environment.yml
+ 1
− 0
View file @ e0c3fc93
......@@ -10,3 +10,4 @@ dependencies:
- pandas
- sqlalchemy
- pyyaml
- networkx
plio/examples/loggers/logging.json
+ 19
− 19
View file @ e0c3fc93
{
"loggers": {
"my_module": {
"level": "DEBUG",
"propagate": false,
"level": "DEBUG",
"propagate": false,
"handlers": [
"console"
]
}
},
"version": 1,
"disable_existing_loggers": false,
},
"version": 1,
"disable_existing_loggers": false,
"handlers": {
"console": {
"formatter": "simple",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
"formatter": "simple",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
"level": "DEBUG"
},
},
"info_file_handler": {
"formatter": "simple",
"backupCount": 10,
"level": "INFO",
"encoding": "utf8",
"class": "logging.handlers.RotatingFileHandler",
"maxBytes": 10485760,
"formatter": "simple",
"backupCount": 10,
"level": "DEBUG",
"encoding": "utf8",
"class": "logging.handlers.RotatingFileHandler",
"maxBytes": 10485760,
"filename": "info.log"
}
},
},
"root": {
"level": "DEBUG",
"level": "DEBUG",
"handlers": [
"console",
"console",
"info_file_handler"
]
},
},
"formatters": {
"simple": {
"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
......
plio/io/io_controlnetwork.py
+ 21
− 17
View file @ e0c3fc93
......@@ -145,7 +145,7 @@ class IsisStore(object):
self._path = path
if not mode:
mode = 'a' # pragma: no cover
mode = 'wb' # pragma: no cover
self._mode = mode
self._handle = None
......@@ -170,13 +170,13 @@ class IsisStore(object):
self._handle.seek(offset)
self._handle.write(data)
def create_points(self, obj, pointid_prefix, pointid_suffix):
def create_points(self, graphs, pointid_prefix, pointid_suffix):
"""
Step through a control network (C) and return protocol buffer point objects
Parameters
----------
obj : list
graphs : list
of iterable objects (dataframes) with the appropriate
attributes: point_id, point_type, serial, measure_type, x, y required.
The entries in the list must support grouping by the point_id attribute.
......@@ -197,20 +197,21 @@ class IsisStore(object):
# TODO: Rewrite using apply syntax for performance
point_sizes = []
point_messages = []
for df in obj:
for i, g in df.groupby('point_id'):
for graph in graphs:
for i, control_point_id in enumerate(graph.point_nodes):
control_point = graph.node[control_point_id]
point_spec = cnf.ControlPointFileEntryV0002()
point_spec.id = _set_pid(self.pointid)
point_spec.id = _set_pid(control_point['pointid'])
for attr, attrtype in self.point_attrs:
if attr in g.columns:
if attr in control_point.keys():
# As per protobuf docs for assigning to a repeated field.
if attr == 'aprioriCovar':
arr = g.iloc[0]['aprioriCovar']
arr = control_point['aprioriCovar']
point_spec.aprioriCovar.extend(arr.ravel().tolist())
else:
setattr(point_spec, attr, attrtype(g.iloc[0][attr]))
point_spec.type = int(g.point_type.iat[0])
# The third argument casts the value to the correct type
setattr(point_spec, attr, attrtype(control_point[attr]))
point_spec.type = int(control_point.get('point_type', 2))
# The reference index should always be the image with the lowest index
point_spec.referenceIndex = 0
......@@ -218,14 +219,19 @@ class IsisStore(object):
# A single extend call is cheaper than many add calls to pack points
measure_iterable = []
for node_id, m in g.iterrows():
for j, control_measure_id in graph.edges(control_point_id):
control_measure = graph.node[control_measure_id]
if control_measure['node_type'] != 'correspondence':
continue
measure_spec = point_spec.Measure()
# For all of the attributes, set if they are an dict accessible attr of the obj.
for attr, attrtype in self.measure_attrs:
if attr in g.columns:
setattr(measure_spec, attr, attrtype(m[attr]))
measure_spec.type = int(m.measure_type)
if attr in control_measure.keys():
setattr(measure_spec, attr, attrtype(control_measure[attr]))
measure_spec.type = int(control_measure['measure_type'])
measure_spec.line = float(control_measure['y'])
measure_spec.sample = float(control_measure['x'])
measure_iterable.append(measure_spec)
self.nmeasures += 1
point_spec.measures.extend(measure_iterable)
......@@ -364,5 +370,3 @@ class IsisStore(object):
if self._handle is not None:
self._handle.close()
self._handle = None
plio/io/io_gdal.py
+ 9
− 5
View file @ e0c3fc93
......@@ -198,6 +198,12 @@ class GeoDataset(object):
self._gcs = self._srs.CloneGeogCS()
return self._srs
@property
def nbands(self):
if not getattr(self, '_nbands', None):
self._nbands = self.dataset.RasterCount
return self._nbands
@property
def geospatial_coordinate_system(self):
if not getattr(self, '_gcs', None):
......@@ -321,10 +327,8 @@ class GeoDataset(object):
@property
def coordinate_transformation(self):
if not getattr(self, '_ct', None):
print('Getting CT')
self._ct = osr.CoordinateTransformation(self.spatial_reference,
self.geospatial_coordinate_system)
print('CT', self._ct)
return self._ct
@property
......@@ -332,7 +336,6 @@ class GeoDataset(object):
if not getattr(self, '_ict', None):
self._ict = osr.CoordinateTransformation(self.geospatial_coordinate_system,
self.spatial_reference)
print(self._ict)
return self._ict
@property
......@@ -558,6 +561,7 @@ def match_rasters(match_to, match_from, destination,
GRA_Cubic, GRA_CubicSpline, GRA_Lanczos, GRA_Average,
GRA_Mode}
"""
import gdalconst # import here so Sphinx can build the docos, mocking is not working
# TODO: If a destination is not provided create an in-memory GeoDataSet object
match_to_srs = match_to.dataset.GetProjection()
......@@ -567,8 +571,8 @@ def match_rasters(match_to, match_from, destination,
match_from__srs = match_from.dataset.GetProjection()
match_from__gt = match_from.geotransform
dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, match_from.RasterCount,
gdalconst.GDT_Float32)
dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, 1,
gdalconst.GDT_Float64)
dst.SetGeoTransform(match_to_gt)
dst.SetProjection(match_to_srs)
......
plio/io/io_krc.py
+ 6
− 3
View file @ e0c3fc93
......@@ -364,9 +364,9 @@ class ReadBin52(object):
self.nseasons)
caseidx = np.repeat(np.arange(self.ncases), self.nseasons)
seasonidx = np.repeat(np.arange(self.nseasons), self.ncases)
flt_seasitems = seasitems.reshape(len(self.vlabels),
flt_seasitems = seasitems.reshape(len(columns),
self.ncases * self.nseasons)
self.seasons = pd.DataFrame(flt_seasitems,
self.seasons = pd.DataFrame(flt_seasitems.T,
index=[caseidx,seasonidx],
columns=columns)
self.seasons.index.names = ['Case', 'Season']
......@@ -432,6 +432,9 @@ class ReadBin52(object):
#Extract the hourly temperature data
hourly2dataframe()
# Extract the seasons
season2dataframe()
# Extract by layer data from the data cube
layeritems = self.bin52data[: , self.ndx: , : , 5: 7, : ]
latitems2dataframe()
......@@ -458,4 +461,4 @@ class ReadTds(object):
self.filename = filename
self.readbin5(headerlen)
print(self.ncases)
assert(self.ncases == self.bin52data.shape[0])
\ No newline at end of file
assert(self.ncases == self.bin52data.shape[0])
plio/io/tests/test_io_controlnetwork.py
+ 25
− 29
View file @ e0c3fc93
......@@ -3,6 +3,7 @@ import sys
import unittest
from time import strftime, gmtime
import networkx as nx
import pandas as pd
import pvl
......@@ -23,38 +24,33 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
cls.serials = ['APOLLO15/METRIC/{}'.format(i) for i in serial_times.values()]
columns = ['point_id', 'point_type', 'serialnumber', 'measure_type', 'x', 'y', 'node_id']
data = []
G = nx.Graph()
G.add_node(0, node_type='image')
G.add_node(1, node_type='image')
G.add_edge(0,1)
G.point_nodes = []
for i in range(cls.npts):
data.append((i, 2, cls.serials[0], 2, 0, 0, 0))
data.append((i, 2, cls.serials[1], 2, 0, 0, 1))
i += 1
G.point_nodes.append(i*100)
G.add_node(i * 100, node_type='point', pointid = i * 100, subpixel=False)
G.add_edge(i*100, 0)
G.add_edge(i*100, 1)
# Add a single correspondence
G.add_node(i * 1000, x=i*.5, y=i*.5, id=i*1000, node_type='correspondence', measure_type=2, serialnumber=cls.serials[0])
G.add_edge(i*1000, i*100)
# Add a second correspondence
G.add_node(i * 2000, x=i*.5, y=i*.5, id=i*2000, node_type='correspondence', measure_type=2, serialnumber=cls.serials[1])
G.add_edge(i * 2000, i*100)
dfs = [pd.DataFrame(data, columns=columns)]
cls.creation_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
cls.modified_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
io_controlnetwork.to_isis('test.net', dfs, mode='wb', targetname='Moon')
io_controlnetwork.to_isis('test.net', [G], mode='wb', targetname='Moon')
cls.header_message_size = 78
cls.point_start_byte = 65609 # 66949
cls.point_start_byte = 65614 # 66949
def test_create_buffer_header(self):
self.npts = 5
serial_times = {295: '1971-07-31T01:24:11.754',
296: '1971-07-31T01:24:36.970'}
self.serials = ['APOLLO15/METRIC/{}'.format(i) for i in serial_times.values()]
columns = ['point_id', 'point_type', 'serialnumber', 'measure_type', 'x', 'y', 'node_id']
data = []
for i in range(self.npts):
data.append((i, 2, self.serials[0], 2, 0, 0, 0))
data.append((i, 2, self.serials[1], 2, 0, 0, 1))
dfs = [pd.DataFrame(data, columns=columns)]
self.creation_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
self.modified_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
io_controlnetwork.to_isis('test.net', dfs, mode='wb', targetname='Moon')
self.header_message_size = 78
self.point_start_byte = 66104 # 66949
......@@ -73,17 +69,16 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
self.assertEqual('None', header_protocol.description)
self.assertEqual(self.modified_date, header_protocol.lastModified)
#Repeating
self.assertEqual([99] * self.npts, header_protocol.pointMessageSizes)
self.assertEqual([137] * self.npts, header_protocol.pointMessageSizes)
def test_create_point(self):
with open('test.net', 'rb') as f:
f.seek(self.point_start_byte)
for i, length in enumerate([99] * self.npts):
for i, length in enumerate(5*[137]):
point_protocol = cnf.ControlPointFileEntryV0002()
raw_point = f.read(length)
point_protocol.ParseFromString(raw_point)
self.assertEqual(str(i), point_protocol.id)
self.assertEqual(str((i + 1) * 100), point_protocol.id)
self.assertEqual(2, point_protocol.type)
for m in point_protocol.measures:
self.assertTrue(m.serialnumber in self.serials)
......@@ -99,11 +94,12 @@ class TestWriteIsisControlNetwork(unittest.TestCase):
self.assertEqual(10, mpoints)
points_bytes = find_in_dict(pvl_header, 'PointsBytes')
self.assertEqual(495, points_bytes)
self.assertEqual(685, points_bytes)
points_start_byte = find_in_dict(pvl_header, 'PointsStartByte')
self.assertEqual(self.point_start_byte, points_start_byte)
@classmethod
def tearDownClass(cls):
os.remove('test.net')
#os.remove('test.net')
pass
setup.py
+ 1
− 2
View file @ e0c3fc93
......@@ -42,11 +42,10 @@ def setup_package():
'pvl',
'protobuf==3.0.0b2',
'h5py',
'icu',
'pandas',
'sqlalchemy',
'pyyaml',
'certifi'],
'networkx'],
classifiers=[
"Development Status :: 3 - Alpha",
"Topic :: Utilities",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment