Merge branch 'master' of https://github.com/USGS-Astrogeology/plio

.travis.yml

@@ -44,7 +44,7 @@ install:
   - conda config --add channels conda-forge
   - conda config --add channels jlaura
   - conda install -c conda-forge gdal h5py
-  - conda install pandas sqlalchemy pyyaml
+  - conda install pandas sqlalchemy pyyaml networkx
   - conda install -c jlaura pvl protobuf
 
   # Development installation

README.rst

@@ -10,13 +10,14 @@ Planetary Input / Output
         :target: https://pypi.python.org/pypi/plio
 
 .. image:: https://travis-ci.org/USGS-Astrogeology/plio.svg?branch=master
-    :target: https://travis-ci.org/USGS-Astrogeology/plio
+   :target: https://travis-ci.org/USGS-Astrogeology/plio
 
-.. image:: https://coveralls.io/repos/github/USGS-Astrogeology/plio/badge.svg?branch=master :target: https://coveralls.io/github/USGS-Astrogeology/plio?branch=master
+.. image:: https://coveralls.io/repos/github/USGS-Astrogeology/plio/badge.svg?branch=master 
+   :target: https://coveralls.io/github/USGS-Astrogeology/plio?branch=master
 
 .. image:: https://readthedocs.org/projects/plio/badge/?version=latest
-    :target: http://plio.readthedocs.io/en/latest/?badge=latest
-    :alt: Documentation Status
+   :target: http://plio.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
 
 
 A planetary file I/O API

appveyor.yml

@@ -53,7 +53,7 @@ install:
     - cmd: conda config --add channels conda-forge
     - cmd: conda config --add channels jlaura
     - cmd: conda install --yes -c conda-forge gdal h5py
-    - cmd: conda install --yes pandas sqlalchemy pyyaml
+    - cmd: conda install --yes pandas sqlalchemy pyyaml networkx
     - cmd: conda install --yes -c jlaura protobuf pvl
 
     # Development installation

conda/build.sh

-#!/bin/bash
-$PYTHON setup.py install

conda/meta.yaml

@@ -2,16 +2,22 @@ package:
   name: plio
   version: 0.1.0
 
-source: 
-  git_url: https://github.com/USGS-Astrogeology/plio.git
+source:
+  path: ../
+
+build:
+  number: 0
+  skip: true  #[win]
+  script: python setup.py install --single-version-externally-managed --record=record.txt
 
 requirements:
-  build: 
-    - python 
+  build:
+    - python
+    - setuptools
     - numpy
     - pvl
     - protobuf 3.0.0b2
-    - gdal >=2
+    - gdal
     - icu
     - h5py
     - pandas
@@ -19,14 +25,21 @@ requirements:
     - pyyaml
   run:
     - python
+    - setuptools
     - numpy
     - pvl
     - protobuf 3.0.0b2
-    - gdal >=2
+    - gdal
     - icu
     - h5py
     - pandas
     - sqlalchemy
     - pyyaml
 
+test:
+  imports:
+    - plio
 
+about:
+  home: http://github.com/USGS-Astrogeology/plio
+  license: Public Domain

environment.yml

@@ -10,3 +10,4 @@ dependencies:
   - pandas
   - sqlalchemy
   - pyyaml
+  - networkx

plio/io/io_autocnetgraph.py

+from io import BytesIO
+import json
+import os
+import warnings
+from zipfile import ZipFile
+
+from networkx.readwrite import json_graph
+import numpy as np
+import pandas as pd
+
+
+try:
+    import autocnet
+    autocnet_avail = True
+except:
+    autocnet_avail = False
+
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        """If input object is an ndarray it will be converted into a dict
+        holding dtype, shape and the data, base64 encoded.
+        """
+        if isinstance(obj, np.ndarray):
+            return dict(__ndarray__= obj.tolist(),
+                        dtype=str(obj.dtype),
+                        shape=obj.shape)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
+
+def save(network, projectname):
+    """
+    Save an AutoCNet candiate graph to disk in a compressed file.  The
+    graph adjacency structure is stored as human readable JSON and all
+    potentially large numpy arrays are stored as compressed binary. The
+    project archive is a standard .zip file that can have any ending,
+    e.g., <projectname>.project, <projectname>.zip, <projectname>.myname.
+
+    TODO: This func. writes a intermediary .npz to disk when saving.  Can
+    we write the .npz to memory?
+
+    Parameters
+    ----------
+    network : object
+              The AutoCNet Candidate Graph object
+
+    projectname : str
+                  The PATH to the output file.
+    """
+    # Convert the graph into json format
+    js = json_graph.node_link_data(network)
+
+    with ZipFile(projectname, 'w') as pzip:
+        js_str = json.dumps(js, cls=NumpyEncoder, sort_keys=True, indent=4)
+        pzip.writestr('graph.json', js_str)
+
+        # Write the array node_attributes to hdf
+        for n, data in network.nodes_iter(data=True):
+            grp = data['node_id']
+            np.savez('{}.npz'.format(data['node_id']),
+                     descriptors=data.descriptors,
+                     _keypoints=data._keypoints,
+                     _keypoints_idx=data._keypoints.index,
+                     _keypoints_columns=data._keypoints.columns)
+            pzip.write('{}.npz'.format(data['node_id']))
+            os.remove('{}.npz'.format(data['node_id']))
+
+        # Write the array edge attributes to hdf
+        for s, d, data in network.edges_iter(data=True):
+            if s > d:
+                s, d = d, s
+            grp = str((s,d))
+            np.savez('{}_{}.npz'.format(s, d),
+                     matches=data.matches,
+                     matches_idx=data.matches.index,
+                     matches_columns=data.matches.columns,
+                     _masks=data._masks,
+                     _masks_idx=data._masks.index,
+                     _masks_columns=data._masks.columns)
+            pzip.write('{}_{}.npz'.format(s, d))
+            os.remove('{}_{}.npz'.format(s, d))
+
+def json_numpy_obj_hook(dct):
+    """Decodes a previously encoded numpy ndarray with proper shape and dtype.
+
+    :param dct: (dict) json encoded ndarray
+    :return: (ndarray) if input was an encoded ndarray
+    """
+    if isinstance(dct, dict) and '__ndarray__' in dct:
+        data = np.asarray(dct['__ndarray__'])
+        return np.frombuffer(data, dct['dtype']).reshape(dct['shape'])
+    return dct
+
+def load(projectname):
+    if autocnet_avail is False:
+        warning.warn('AutoCNet Library is not available.  Unable to load an AutoCNet CandidateGraph')
+        return
+
+    with ZipFile(projectname, 'r') as pzip:
+        # Read the graph object
+        with pzip.open('graph.json', 'r') as g:
+            data = json.loads(g.read().decode(),object_hook=json_numpy_obj_hook)
+
+        cg = autocnet.graph.network.CandidateGraph()
+        Edge = autocnet.graph.edge.Edge
+        Node = autocnet.graph.node.Node
+        # Reload the graph attributes
+        cg.graph = data['graph']
+        # Handle nodes
+        for d in data['nodes']:
+            n = Node(image_name=d['image_name'], image_path=d['image_path'], node_id=d['id'])
+            n['hash'] = d['hash']
+            # Load the byte stream for the nested npz file into memory and then unpack
+            nzf = np.load(BytesIO(pzip.read('{}.npz'.format(d['id']))))
+            n._keypoints = pd.DataFrame(nzf['_keypoints'], index=nzf['_keypoints_idx'], columns=nzf['_keypoints_columns'])
+            n.descriptors = nzf['descriptors']
+            cg.add_node(d['node_id'])
+            cg.node[d['node_id']] = n
+        for e in data['links']:
+            cg.add_edge(e['source'], e['target'])
+            edge = Edge()
+            edge.source = cg.node[e['source']]
+            edge.destination = cg.node[e['target']]
+            edge['fundamental_matrix'] = e['fundamental_matrix']
+            edge['weight'] = e['weight']
+            nzf = np.load(BytesIO(pzip.read('{}_{}.npz'.format(e['source'], e['target']))))
+
+            edge._masks = pd.DataFrame(nzf['_masks'], index=nzf['_masks_idx'], columns=nzf['_masks_columns'])
+            edge.matches = pd.DataFrame(nzf['matches'], index=nzf['matches_idx'], columns=nzf['matches_columns'])
+            # Add a mock edge
+            cg.edge[e['source']][e['target']] = edge
+
+    return cg

plio/io/io_controlnetwork.py

@@ -222,10 +222,12 @@ class IsisStore(object):
                     measure_spec = point_spec.Measure()
                     # For all of the attributes, set if they are an dict accessible attr of the obj.
                     for attr, attrtype in self.measure_attrs:
+
                         if attr in g.columns:
                             setattr(measure_spec, attr, attrtype(m[attr]))
                     measure_spec.type = int(m.measure_type)
-
+                    measure_spec.line = m.y
+                    measure_spec.sample = m.x
                     measure_iterable.append(measure_spec)
                     self.nmeasures += 1
                 point_spec.measures.extend(measure_iterable)
@@ -364,5 +366,3 @@ class IsisStore(object):
         if self._handle is not None:
             self._handle.close()
         self._handle = None
-
-

plio/io/io_gdal.py

@@ -52,7 +52,7 @@ class GeoDataset(object):
                  The bounding box of the image in lat/lon space
 
     geotransform : object
-                   Geotransform reference OGR object as an array of size 6 containing the affine 
+                   Geotransform reference OGR object as an array of size 6 containing the affine
                    transformation coefficients for transforming from raw sample/line to projected x/y.
                    xproj = geotransform[0] + sample * geotransform[1] + line * geotransform[2]
                    yproj = geotransform[3] + sample * geotransform[4] + line * geotransform[5]
@@ -61,7 +61,7 @@ class GeoDataset(object):
                                    Geospatial coordinate system OSR object.
 
     latlon_extent : list
-                    of two tuples containing the latitide/longitude boundaries. 
+                    of two tuples containing the latitide/longitude boundaries.
                     This list is in the form [(lowerlat, lowerlon), (upperlat, upperlon)].
 
     pixel_width : float
@@ -87,9 +87,9 @@ class GeoDataset(object):
                 Note: This is the third value geotransform array.
 
     xy_extent : list
-                of two tuples containing the sample/line boundaries. 
-                The first value is the upper left corner of the upper left pixel and 
-                the second value is the lower right corner of the lower right pixel. 
+                of two tuples containing the sample/line boundaries.
+                The first value is the upper left corner of the upper left pixel and
+                the second value is the lower right corner of the lower right pixel.
                 This list is in the form [(minx, miny), (maxx, maxy)].
 
     xy_corners : list
@@ -101,26 +101,26 @@ class GeoDataset(object):
                  Note: This is the fifth value geotransform array.
 
     coordinate_transformation : object
-                                The coordinate transformation from the spatial reference system to 
+                                The coordinate transformation from the spatial reference system to
                                 the geospatial coordinate system.
-        
+
     inverse_coordinate_transformation : object
-                                        The coordinate transformation from the geospatial 
+                                        The coordinate transformation from the geospatial
                                         coordinate system to the spatial reference system.
-        
+
     scale : tuple
-            The name and value of the linear projection units of the spatial reference system. 
+            The name and value of the linear projection units of the spatial reference system.
             This tuple is of type string/float of the form (unit name, value).
             To transform a linear distance to meters, multiply by this value.
             If no units are available ("Meters", 1) will be returned.
-                 
+
     spheroid : tuple
-               The spheroid found in the metadata using the spatial reference system. 
+               The spheroid found in the metadata using the spatial reference system.
                This is of the form (semi-major, semi-minor, inverse flattening).
 
     raster_size : tuple
                   The dimensions of the raster, i.e. (number of samples, number of lines).
-        
+
     central_meridian : float
                        The central meridian of the map projection from the metadata.
 
@@ -382,7 +382,7 @@ class GeoDataset(object):
         -------
         lat, lon : tuple
                    (Latitude, Longitude) corresponding to the given (x,y).
-        
+
         """
         try:
             geotransform = self.geotransform
@@ -410,7 +410,7 @@ class GeoDataset(object):
         -------
         x, y : tuple
                (Sample, line) position corresponding to the given (latitude, longitude).
-        
+
         """
         geotransform = self.geotransform
         upperlat, upperlon, _ = self.inverse_coordinate_transformation.TransformPoint(lon, lat)
@@ -567,7 +567,8 @@ def match_rasters(match_to, match_from, destination,
     match_from__srs = match_from.dataset.GetProjection()
     match_from__gt = match_from.geotransform
 
-    dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, 1, gdalconst.GDT_Float32)
+    dst = gdal.GetDriverByName('GTiff').Create(destination, width, height, match_from.RasterCount,
+                                               gdalconst.GDT_Float32)
     dst.SetGeoTransform(match_to_gt)
     dst.SetProjection(match_to_srs)
 

plio/io/io_spectral_profiler.py

+import os
 import pandas as pd
 import pvl
 import numpy as np
 
 from plio.utils.utils import find_in_dict
-
+from plio.io.io_gdal import GeoDataset
 
 class Spectral_Profiler(object):
 
@@ -52,6 +53,7 @@ class Spectral_Profiler(object):
 
         label = pvl.load(input_data)
         self.label = label
+        self.input_data = input_data
         with open(input_data, 'rb') as indata:
             # Extract and handle the ancillary data
             ancillary_data = find_in_dict(label, "ANCILLARY_AND_SUPPLEMENT_DATA")
@@ -128,3 +130,20 @@ class Spectral_Profiler(object):
                     self.spectra[i] = self.spectra[i][self.spectra[i]['QA'] < qa_threshold]
 
             self.spectra = pd.Panel(self.spectra)
+
+    def open_browse(self, extension='.jpg'):
+        """
+        Attempt to open the browse image corresponding to the spc file
+
+        Parameters
+        ----------
+        extension : str
+                    The file type extension to be added to the base name
+                    of the spc file.
+
+        Returns
+        -------
+
+        """
+        path, ext = os.path.splitext(self.input_data)
+        self.browse = GeoDataset(path + extension)

plio/io/tests/test_io_spectral_profiler.py

@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.abspath('..'))
 
 from plio.examples import get_path
 from plio.io import io_spectral_profiler
-
+from plio.io.io_gdal import GeoDataset
 
 class Test_Spectral_Profiler_IO(unittest.TestCase):
     
@@ -21,6 +21,11 @@ class Test_Spectral_Profiler_IO(unittest.TestCase):
         self.assertIsInstance(ds.spectra, pd.Panel)
         self.assertEqual(ds.spectra[0].columns.tolist(), ['RAW', 'REF1', 'REF2', 'QA'])
 
+    def test_read_browse(self):
+        ds = io_spectral_profiler.Spectral_Profiler(self.examplefile)
+        ds.open_browse()
+        self.assertIsInstance(ds.browse, GeoDataset)
+        self.assertEqual(ds.browse.read_array().shape, (512, 456))
 
 if __name__ == '__main__':
     unittest.main()

setup.py

@@ -20,7 +20,7 @@ def setup_package():
         else:
             glob_name = 'examples/' + i + '/*'
         examples.add(glob_name)
-    
+
     setup(
         name = "plio",
         version = VERSION,
@@ -37,13 +37,16 @@ def setup_package():
                 ['sqlalchemy_json/*.py', 'sqlalchemy_json/LICENSE']},
         zip_safe=False,
         install_requires=[
-            'gdal>=2',
+            'gdal',
+            'numpy',
             'pvl',
             'protobuf==3.0.0b2',
             'h5py',
+            'icu',
             'pandas',
             'sqlalchemy',
-            'pyyaml'],
+            'pyyaml',
+            'certifi'],
         classifiers=[
             "Development Status :: 3 - Alpha",
             "Topic :: Utilities",