diff --git a/.travis.yml b/.travis.yml
index 5d4c074ddb90a7e1fbe45f24f16a30c3732054cf..027346f8311408e148fbbcf3870bb0ed67090061 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -44,7 +44,7 @@ install:
- conda config --add channels conda-forge
- conda config --add channels jlaura
- conda install -c conda-forge gdal h5py
- - conda install pandas sqlalchemy pyyaml
+ - conda install pandas sqlalchemy pyyaml networkx
- conda install -c jlaura pvl protobuf
# Development installation
diff --git a/README.rst b/README.rst
index 4ca3f3ff4d0fabcbf1138c78607ff6e87362864d..c15a55c5faa782be65cce5c594ccf38aff2ddfdc 100644
--- a/README.rst
+++ b/README.rst
@@ -10,13 +10,14 @@ Planetary Input / Output
:target: https://pypi.python.org/pypi/plio
.. image:: https://travis-ci.org/USGS-Astrogeology/plio.svg?branch=master
- :target: https://travis-ci.org/USGS-Astrogeology/plio
+ :target: https://travis-ci.org/USGS-Astrogeology/plio
-.. image:: https://coveralls.io/repos/github/USGS-Astrogeology/plio/badge.svg?branch=master :target: https://coveralls.io/github/USGS-Astrogeology/plio?branch=master
+.. image:: https://coveralls.io/repos/github/USGS-Astrogeology/plio/badge.svg?branch=master
+ :target: https://coveralls.io/github/USGS-Astrogeology/plio?branch=master
.. image:: https://readthedocs.org/projects/plio/badge/?version=latest
- :target: http://plio.readthedocs.io/en/latest/?badge=latest
- :alt: Documentation Status
+ :target: http://plio.readthedocs.io/en/latest/?badge=latest
+ :alt: Documentation Status
A planetary file I/O API
diff --git a/appveyor.yml b/appveyor.yml
index ad091115a9002a4b2559a563cdb01681c284b9c2..47dedfaf6f75f9393f5a874551481630c1d5e354 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -53,7 +53,7 @@ install:
- cmd: conda config --add channels conda-forge
- cmd: conda config --add channels jlaura
- cmd: conda install --yes -c conda-forge gdal h5py
- - cmd: conda install --yes pandas sqlalchemy pyyaml
+ - cmd: conda install --yes pandas sqlalchemy pyyaml networkx
- cmd: conda install --yes -c jlaura protobuf pvl
# Development installation
diff --git a/conda/build.sh b/conda/build.sh
deleted file mode 100755
index 4dad93a1c02abe34b4326e6712b9d046abd2f4a8..0000000000000000000000000000000000000000
--- a/conda/build.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/bash
-$PYTHON setup.py install
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 08523fa178075ebb9be9e4e9e21319cab176a844..18fbf121a501256f19dd85b24621c628de4ef3d7 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -2,16 +2,22 @@ package:
name: plio
version: 0.1.0
-source:
- git_url: https://github.com/USGS-Astrogeology/plio.git
+source:
+ path: ../
+
+build:
+ number: 0
+ skip: true #[win]
+ script: python setup.py install --single-version-externally-managed --record=record.txt
requirements:
- build:
- - python
+ build:
+ - python
+ - setuptools
- numpy
- pvl
- protobuf 3.0.0b2
- - gdal >=2
+ - gdal
- icu
- h5py
- pandas
@@ -19,14 +25,21 @@ requirements:
- pyyaml
run:
- python
+ - setuptools
- numpy
- pvl
- protobuf 3.0.0b2
- - gdal >=2
+ - gdal
- icu
- h5py
- pandas
- sqlalchemy
- pyyaml
+test:
+ imports:
+ - plio
+about:
+ home: http://github.com/USGS-Astrogeology/plio
+ license: Public Domain
diff --git a/environment.yml b/environment.yml
index c9752688e6a26f7550601269b160d255cc27df5c..f51c74af88f78d0d7468b559581c862cc1172c07 100644
--- a/environment.yml
+++ b/environment.yml
@@ -10,3 +10,4 @@ dependencies:
- pandas
- sqlalchemy
- pyyaml
+ - networkx
diff --git a/plio/io/io_autocnetgraph.py b/plio/io/io_autocnetgraph.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1360aba6acd26275cfe29f6c1d0de71653a72e1
--- /dev/null
+++ b/plio/io/io_autocnetgraph.py
@@ -0,0 +1,132 @@
+from io import BytesIO
+import json
+import os
+import warnings
+from zipfile import ZipFile
+
+from networkx.readwrite import json_graph
+import numpy as np
+import pandas as pd
+
+
+try:
+ import autocnet
+ autocnet_avail = True
+except:
+ autocnet_avail = False
+
+class NumpyEncoder(json.JSONEncoder):
+ def default(self, obj):
+ """If input object is an ndarray it will be converted into a dict
+ holding dtype, shape and the data, base64 encoded.
+ """
+ if isinstance(obj, np.ndarray):
+ return dict(__ndarray__= obj.tolist(),
+ dtype=str(obj.dtype),
+ shape=obj.shape)
+ # Let the base class default method raise the TypeError
+ return json.JSONEncoder.default(self, obj)
+
+def save(network, projectname):
+ """
+ Save an AutoCNet candiate graph to disk in a compressed file. The
+ graph adjacency structure is stored as human readable JSON and all
+ potentially large numpy arrays are stored as compressed binary. The
+ project archive is a standard .zip file that can have any ending,
+ e.g., <projectname>.project, <projectname>.zip, <projectname>.myname.
+
+ TODO: This func. writes a intermediary .npz to disk when saving. Can
+ we write the .npz to memory?
+
+ Parameters
+ ----------
+ network : object
+ The AutoCNet Candidate Graph object
+
+ projectname : str
+ The PATH to the output file.
+ """
+ # Convert the graph into json format
+ js = json_graph.node_link_data(network)
+
+ with ZipFile(projectname, 'w') as pzip:
+ js_str = json.dumps(js, cls=NumpyEncoder, sort_keys=True, indent=4)
+ pzip.writestr('graph.json', js_str)
+
+ # Write the array node_attributes to hdf
+ for n, data in network.nodes_iter(data=True):
+ grp = data['node_id']
+ np.savez('{}.npz'.format(data['node_id']),
+ descriptors=data.descriptors,
+ _keypoints=data._keypoints,
+ _keypoints_idx=data._keypoints.index,
+ _keypoints_columns=data._keypoints.columns)
+ pzip.write('{}.npz'.format(data['node_id']))
+ os.remove('{}.npz'.format(data['node_id']))
+
+ # Write the array edge attributes to hdf
+ for s, d, data in network.edges_iter(data=True):
+ if s > d:
+ s, d = d, s
+ grp = str((s,d))
+ np.savez('{}_{}.npz'.format(s, d),
+ matches=data.matches,
+ matches_idx=data.matches.index,
+ matches_columns=data.matches.columns,
+ _masks=data._masks,
+ _masks_idx=data._masks.index,
+ _masks_columns=data._masks.columns)
+ pzip.write('{}_{}.npz'.format(s, d))
+ os.remove('{}_{}.npz'.format(s, d))
+
+def json_numpy_obj_hook(dct):
+ """Decodes a previously encoded numpy ndarray with proper shape and dtype.
+
+ :param dct: (dict) json encoded ndarray
+ :return: (ndarray) if input was an encoded ndarray
+ """
+ if isinstance(dct, dict) and '__ndarray__' in dct:
+ data = np.asarray(dct['__ndarray__'])
+ return np.frombuffer(data, dct['dtype']).reshape(dct['shape'])
+ return dct
+
+def load(projectname):
+ if autocnet_avail is False:
+ warning.warn('AutoCNet Library is not available. Unable to load an AutoCNet CandidateGraph')
+ return
+
+ with ZipFile(projectname, 'r') as pzip:
+ # Read the graph object
+ with pzip.open('graph.json', 'r') as g:
+ data = json.loads(g.read().decode(),object_hook=json_numpy_obj_hook)
+
+ cg = autocnet.graph.network.CandidateGraph()
+ Edge = autocnet.graph.edge.Edge
+ Node = autocnet.graph.node.Node
+ # Reload the graph attributes
+ cg.graph = data['graph']
+ # Handle nodes
+ for d in data['nodes']:
+ n = Node(image_name=d['image_name'], image_path=d['image_path'], node_id=d['id'])
+ n['hash'] = d['hash']
+ # Load the byte stream for the nested npz file into memory and then unpack
+ nzf = np.load(BytesIO(pzip.read('{}.npz'.format(d['id']))))
+ n._keypoints = pd.DataFrame(nzf['_keypoints'], index=nzf['_keypoints_idx'], columns=nzf['_keypoints_columns'])
+ n.descriptors = nzf['descriptors']
+ cg.add_node(d['node_id'])
+ cg.node[d['node_id']] = n
+ for e in data['links']:
+ cg.add_edge(e['source'], e['target'])
+ edge = Edge()
+ edge.source = cg.node[e['source']]
+ edge.destination = cg.node[e['target']]
+ edge['fundamental_matrix'] = e['fundamental_matrix']
+ edge['weight'] = e['weight']
+ nzf = np.load(BytesIO(pzip.read('{}_{}.npz'.format(e['source'], e['target']))))
+
+ edge._masks = pd.DataFrame(nzf['_masks'], index=nzf['_masks_idx'], columns=nzf['_masks_columns'])
+ edge.matches = pd.DataFrame(nzf['matches'], index=nzf['matches_idx'], columns=nzf['matches_columns'])
+ # Add a mock edge
+ cg.edge[e['source']][e['target']] = edge
+
+ return cg
diff --git a/plio/io/io_controlnetwork.py b/plio/io/io_controlnetwork.py
index afcf5a48b6f197788a7458c7af11af3356371974..b120bd6a89923e3a09e710c430cf6f280605689c 100644
--- a/plio/io/io_controlnetwork.py
+++ b/plio/io/io_controlnetwork.py
@@ -222,10 +222,12 @@ class IsisStore(object):
measure_spec = point_spec.Measure()
# For all of the attributes, set if they are an dict accessible attr of the obj.
for attr, attrtype in self.measure_attrs:
+
if attr in g.columns:
setattr(measure_spec, attr, attrtype(m[attr]))
measure_spec.type = int(m.measure_type)
-
+ measure_spec.line = m.y
+ measure_spec.sample = m.x
measure_iterable.append(measure_spec)
self.nmeasures += 1
point_spec.measures.extend(measure_iterable)
@@ -364,5 +366,3 @@ class IsisStore(object):
if self._handle is not None:
self._handle.close()
self._handle = None
-
-
diff --git a/plio/io/io_gdal.py b/plio/io/io_gdal.py
index 555f73ebe17d61838b74e4a5286c4f038b9caeba..372b118f3da56a8324d2f23f13077a4df7955360 100644
--- a/plio/io/io_gdal.py
+++ b/plio/io/io_gdal.py
@@ -52,7 +52,7 @@ class GeoDataset(object):
The bounding box of the image in lat/lon space
geotransform : object
- Geotransform reference OGR object as an array of size 6 containing the affine
+ Geotransform reference OGR object as an array of size 6 containing the affine
transformation coefficients for transforming from raw sample/line to projected x/y.
xproj = geotransform[0] + sample * geotransform[1] + line * geotransform[2]
yproj = geotransform[3] + sample * geotransform[4] + line * geotransform[5]
@@ -61,7 +61,7 @@ class GeoDataset(object):
Geospatial coordinate system OSR object.
latlon_extent : list
- of two tuples containing the latitide/longitude boundaries.
+ of two tuples containing the latitide/longitude boundaries.
This list is in the form [(lowerlat, lowerlon), (upperlat, upperlon)].
pixel_width : float
@@ -87,9 +87,9 @@ class GeoDataset(object):
Note: This is the third value geotransform array.
xy_extent : list
- of two tuples containing the sample/line boundaries.
- The first value is the upper left corner of the upper left pixel and
- the second value is the lower right corner of the lower right pixel.
+ of two tuples containing the sample/line boundaries.
+ The first value is the upper left corner of the upper left pixel and
+ the second value is the lower right corner of the lower right pixel.
This list is in the form [(minx, miny), (maxx, maxy)].
xy_corners : list
@@ -101,26 +101,26 @@ class GeoDataset(object):
Note: This is the fifth value geotransform array.
coordinate_transformation : object
- The coordinate transformation from the spatial reference system to
+ The coordinate transformation from the spatial reference system to
the geospatial coordinate system.
-
+
inverse_coordinate_transformation : object
- The coordinate transformation from the geospatial
+ The coordinate transformation from the geospatial
coordinate system to the spatial reference system.
-
+
scale : tuple
- The name and value of the linear projection units of the spatial reference system.
+ The name and value of the linear projection units of the spatial reference system.
This tuple is of type string/float of the form (unit name, value).
To transform a linear distance to meters, multiply by this value.
If no units are available ("Meters", 1) will be returned.
-
+
spheroid : tuple
- The spheroid found in the metadata using the spatial reference system.
+ The spheroid found in the metadata using the spatial reference system.
This is of the form (semi-major, semi-minor, inverse flattening).
raster_size : tuple
The dimensions of the raster, i.e. (number of samples, number of lines).
-
+
central_meridian : float
The central meridian of the map projection from the metadata.
@@ -382,7 +382,7 @@ class GeoDataset(object):
-------
lat, lon : tuple
(Latitude, Longitude) corresponding to the given (x,y).
-
+
"""
try:
geotransform = self.geotransform
@@ -410,7 +410,7 @@ class GeoDataset(object):
-------
x, y : tuple
(Sample, line) position corresponding to the given (latitude, longitude).
-
+
"""
geotransform = self.geotransform
upperlat, upperlon, _ = self.inverse_coordinate_transformation.TransformPoint(lon, lat)
@@ -567,7 +567,8 @@ def match_rasters(match_to, match_from, destination,
match_from__srs = match_from.dataset.GetProjection()
match_from__gt = match_from.geotransform
- dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, 1, gdalconst.GDT_Float32)
+ dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, match_from.RasterCount,
+ gdalconst.GDT_Float32)
dst.SetGeoTransform(match_to_gt)
dst.SetProjection(match_to_srs)
diff --git a/plio/io/io_spectral_profiler.py b/plio/io/io_spectral_profiler.py
index 0619eb649ac18b59758e01ad399d3d22ba1dcdf6..98cb989130fb5d871f4547528f7266b6f78dbdd3 100755
--- a/plio/io/io_spectral_profiler.py
+++ b/plio/io/io_spectral_profiler.py
@@ -1,9 +1,10 @@
+import os
import pandas as pd
import pvl
import numpy as np
from plio.utils.utils import find_in_dict
-
+from plio.io.io_gdal import GeoDataset
class Spectral_Profiler(object):
@@ -52,6 +53,7 @@ class Spectral_Profiler(object):
label = pvl.load(input_data)
self.label = label
+ self.input_data = input_data
with open(input_data, 'rb') as indata:
# Extract and handle the ancillary data
ancillary_data = find_in_dict(label, "ANCILLARY_AND_SUPPLEMENT_DATA")
@@ -128,3 +130,20 @@ class Spectral_Profiler(object):
self.spectra[i] = self.spectra[i][self.spectra[i]['QA'] < qa_threshold]
self.spectra = pd.Panel(self.spectra)
+
+ def open_browse(self, extension='.jpg'):
+ """
+ Attempt to open the browse image corresponding to the spc file
+
+ Parameters
+ ----------
+ extension : str
+ The file type extension to be added to the base name
+ of the spc file.
+
+ Returns
+ -------
+
+ """
+ path, ext = os.path.splitext(self.input_data)
+ self.browse = GeoDataset(path + extension)
diff --git a/plio/io/tests/test_io_spectral_profiler.py b/plio/io/tests/test_io_spectral_profiler.py
index 61127a6053936d90212d0841b4948b7cf489897a..3d05e05c4241e3fbc900aa9579072238a3bb06d6 100644
--- a/plio/io/tests/test_io_spectral_profiler.py
+++ b/plio/io/tests/test_io_spectral_profiler.py
@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.abspath('..'))
from plio.examples import get_path
from plio.io import io_spectral_profiler
-
+from plio.io.io_gdal import GeoDataset
class Test_Spectral_Profiler_IO(unittest.TestCase):
@@ -21,6 +21,11 @@ class Test_Spectral_Profiler_IO(unittest.TestCase):
self.assertIsInstance(ds.spectra, pd.Panel)
self.assertEqual(ds.spectra[0].columns.tolist(), ['RAW', 'REF1', 'REF2', 'QA'])
+ def test_read_browse(self):
+ ds = io_spectral_profiler.Spectral_Profiler(self.examplefile)
+ ds.open_browse()
+ self.assertIsInstance(ds.browse, GeoDataset)
+ self.assertEqual(ds.browse.read_array().shape, (512, 456))
if __name__ == '__main__':
unittest.main()
diff --git a/setup.py b/setup.py
index 61e417d15fadcea8592a3691c343ae4f66a3eb05..ca82517062677bfeed26e5018d5a51770cf169a7 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ def setup_package():
else:
glob_name = 'examples/' + i + '/*'
examples.add(glob_name)
-
+
setup(
name = "plio",
version = VERSION,
@@ -37,13 +37,16 @@ def setup_package():
['sqlalchemy_json/*.py', 'sqlalchemy_json/LICENSE']},
zip_safe=False,
install_requires=[
- 'gdal>=2',
+ 'gdal',
+ 'numpy',
'pvl',
'protobuf==3.0.0b2',
'h5py',
+ 'icu',
'pandas',
'sqlalchemy',
- 'pyyaml'],
+ 'pyyaml',
+ 'certifi'],
classifiers=[
"Development Status :: 3 - Alpha",
"Topic :: Utilities",