From 12c1aede216d468d474172fb2127778e487bc15a Mon Sep 17 00:00:00 2001
From: Jay Laura <jlaura@usgs.gov>
Date: Fri, 28 Jul 2023 07:28:47 -0700
Subject: [PATCH] Updates for # in PVL. (#195)
* Fixes 194 and simplifies serial generation.
* Updates CTX label to include a comment.
* Fixces SQLAlchemy Bind for pytests
* Updates GDAL dependency
* Fixes pandas import for frame_equal test
* Fixes isis serial number generation.
* Fixes proj string on test data.
* Fixes conditional imports for gdal, osr, ogr.
* Fixes projection test for gdal > 3.
* Upates CHANGELOG to include testing fixes.
---
CHANGELOG.md | 3 +
environment.yml | 4 +-
plio/examples/CTX/ctx.pvl | 2 +
...s_MGS_MOLA_ClrShade_MAP2_90.0N0.0_POLA.tif | Bin 9280 -> 10510 bytes
plio/io/__init__.py | 20 +----
plio/io/extract_metadata.py | 5 +-
plio/io/io_db.py | 9 +-
plio/io/io_gdal.py | 4 +-
plio/io/isis_serial_number.py | 82 ++----------------
plio/io/tests/test_io_bae.py | 2 +-
plio/io/tests/test_io_gdal.py | 14 +--
plio/io/tests/test_metadata.py | 4 +-
12 files changed, 37 insertions(+), 112 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58b34bd..ab23f36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,9 @@ release.
## [Unreleased]
### Fixed
+- Tests for gdal > 3 and pvl > 1.0. This includes fixing the `k` value on the MOLA polar stereographic test data and updating the proj string for GDAL > 3 (new fields are included).
+- Conditional GDAL import to support gdal > 3.0
+- `generate_isis_serial` to work on cubes that have been run through `jigsaw` by removing the custom `SerialNumberDecoder`. Fixes [#194](https://github.com/DOI-USGS/plio/issues/194)
- Updated `create_pvl_header()` to add a newline (`\n`) character to the end of the pvl string. This ensures that the control networks can be written, then read back in using pvl 1.3.0 [#193](https://github.com/USGS-Astrogeology/plio/pull/193)
## [1.5.3]()
diff --git a/environment.yml b/environment.yml
index 37ea027..233b9cc 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,10 +1,8 @@
name: plio
channels:
- conda-forge
- - usgs-astrogeology
dependencies:
- - libgdal < 3
- - gdal < 3
+ - gdal
- numpy
- pyproj
- h5py
diff --git a/plio/examples/CTX/ctx.pvl b/plio/examples/CTX/ctx.pvl
index b062798..a44e44c 100644
--- a/plio/examples/CTX/ctx.pvl
+++ b/plio/examples/CTX/ctx.pvl
@@ -67,6 +67,8 @@ Object = IsisCube
End_Group
End_Object
+# Jigged
+
Object = Label
Bytes = 65536
End_Object
diff --git a/plio/examples/Projections/Mars_MGS_MOLA_ClrShade_MAP2_90.0N0.0_POLA.tif b/plio/examples/Projections/Mars_MGS_MOLA_ClrShade_MAP2_90.0N0.0_POLA.tif
index e47b318148f7c268ebf1aa4e20ee7051695bf24c..eed8e641775cca2fcef4ab8dba81782982a2d73c 100644
GIT binary patch
delta 324
zcmX@$(HF$#>8Zuwpt6z8N@XI48xxz_!~i8GIn{|JN_Owm7#R4Pc^H@(7#NOeFfcT=
z@G!73FfiQHU|?9^&dd<Pz`#(U&cLvtori&wfq_9mgMndVI}<}J0|P^b#>Oi-Op`m9
zWrR5x7#J8h7#J8C{?{{dY-|r<;F!FfSzeHhftzDPJ0k--12357nEaesU6GN2iw7ji
z!oUrtSs6sYG&6%Zm}Z)+&mzzC!G3Zevq-)EOHW7bwI}@N+oU>xYysiYyzIRE^1K>%
z=U@fD#N1Q`TLqYqOJYfBE)w50CnvL@I6pH5Ni3i!GdC5aC@3{8wJ0?&IaMJywJ0+s
WGcm8G24p@11B2Kmh;5U7nXLf`22r8_
delta 21
ccmeARI^e<K>8ZuQ!N9;U*+Io^qu(cG06)P71^@s6
diff --git a/plio/io/__init__.py b/plio/io/__init__.py
index 7c0f093..b67fa43 100644
--- a/plio/io/__init__.py
+++ b/plio/io/__init__.py
@@ -1,25 +1,13 @@
# Conditional imports for GDAL
import importlib
import warnings
-import sys
try:
- gdal = importlib.util.find_spec('gdal')
- ogr = importlib.util.find_spec('osgeo.ogr')
- osr = importlib.util.find_spec('osr')
-
- gdal = gdal.loader.load_module()
- ogr = ogr.loader.load_module()
- osr = osr.loader.load_module()
- gdal.UseExceptions()
+ gdal = importlib.import_module('osgeo.gdal')
+ osr = importlib.import_module('osgeo.osr')
+ ogr = importlib.import_module('osgeo.ogr')
except:
- try:
- gdal = importlib.util.find_spec('osgeo.gdal')
- gdal = gdal.loader.load_module()
- except:
- gdal = None
- ogr = None
- osr = None
+ gdal = osr = ogr = None
def conditional_gdal(func):
def has_gdal(*args, **kwargs):
diff --git a/plio/io/extract_metadata.py b/plio/io/extract_metadata.py
index 6d94c2a..ea1e370 100644
--- a/plio/io/extract_metadata.py
+++ b/plio/io/extract_metadata.py
@@ -1,5 +1,6 @@
-from plio.io import osr, conditional_gdal
-
+
+from plio.io import conditional_gdal, osr
+
import_options = ['ImportFromWkt', 'ImportFromProj4',
'ImportFromEPSG', 'ImportFromUSGS',
'ImportFromXML']
diff --git a/plio/io/io_db.py b/plio/io/io_db.py
index 988ec58..5903484 100644
--- a/plio/io/io_db.py
+++ b/plio/io/io_db.py
@@ -1,9 +1,10 @@
from sqlalchemy import Column, Integer, String, create_engine, orm
-from sqlalchemy.ext import declarative
+from sqlalchemy.orm import declarative_base
+from sqlalchemy import MetaData
from plio.sqlalchemy_json.alchemy import NestedJsonObject
-Base = declarative.declarative_base()
+Base = declarative_base()
def setup_db_session(db):
@@ -21,8 +22,8 @@ def setup_db_session(db):
A SQLAlchemy session object
"""
engine = create_engine('sqlite:///{}'.format(db))
- Base.metadata.bind = engine
- Base.metadata.create_all()
+ meta = MetaData()
+ meta.create_all(engine)
return orm.sessionmaker(bind=engine)()
diff --git a/plio/io/io_gdal.py b/plio/io/io_gdal.py
index fc4222a..749fa40 100644
--- a/plio/io/io_gdal.py
+++ b/plio/io/io_gdal.py
@@ -7,11 +7,9 @@ import affine
import numpy as np
import pvl
-
-from plio.io import extract_metadata, conditional_gdal
from plio.geofuncs import geofuncs
from plio.utils.utils import find_in_dict
-from plio.io import gdal, ogr, osr
+from plio.io import gdal, ogr, osr, extract_metadata, conditional_gdal
NP2GDAL_CONVERSION = {
"byte": 1,
diff --git a/plio/io/isis_serial_number.py b/plio/io/isis_serial_number.py
index faba3bf..725e9a1 100644
--- a/plio/io/isis_serial_number.py
+++ b/plio/io/isis_serial_number.py
@@ -1,3 +1,4 @@
+import datetime
import warnings
import pvl
@@ -71,7 +72,7 @@ def generate_serial_number(label):
The ISIS compatible serial number
"""
if not isinstance(label, PVLModule):
- label = pvl.load(label, decoder=SerialNumberDecoder())
+ label = pvl.load(label)
# Get the translation information
translation = get_isis_translation(label)
@@ -97,82 +98,11 @@ def generate_serial_number(label):
serial_entry = search_translation[serial_entry]
elif '*' in search_translation.keys() and search_translation['*'] != '*':
serial_entry = search_translation['*']
+ if isinstance(serial_entry, datetime.datetime):
+ # PVL returns datetime objects now. Convert these to string and strip trailing zeros on microseconds.
+ serial_entry = serial_entry.strftime('%Y-%m-%dT%H:%M:%S.%f').rstrip('0')
serial_number.append(serial_entry)
except:
pass
-
+
return '/'.join(serial_number)
-
-
-class SerialNumberDecoder(pvl.decoder.PVLDecoder):
- """
- A PVL Decoder class to handle cube label parsing for the purpose of creating a valid ISIS
- serial number. Inherits from the PVLDecoder in planetarypy's pvl module.
- """
-
- def decode_simple_value(self, value: str):
- """Returns a Python object based on *value*, assuming
- that *value* can be decoded as a PVL Simple Value::
-
- <Simple-Value> ::= (<Numeric> | <String>)
-
- Modified from https://pvl.readthedocs.io/en/stable/_modules/pvl/decoder.html#PVLDecoder.decode_simple_value
- Modification entails stripping datetime from list of functions.
- """
- for d in (
- self.decode_quoted_string,
- self.decode_non_decimal,
- self.decode_decimal,
- ):
- try:
- return d(value)
- except ValueError:
- pass
-
- if value.casefold() == self.grammar.none_keyword.casefold():
- return None
-
- if value.casefold() == self.grammar.true_keyword.casefold():
- return True
-
- if value.casefold() == self.grammar.false_keyword.casefold():
- return False
-
- return self.decode_unquoted_string(value)
-
- def decode_unquoted_string(self, value: str) -> str:
- """Returns a Python ``str`` if *value* can be decoded
- as an unquoted string, based on this decoder's grammar.
- Raises a ValueError otherwise.
-
- Modified from: https://pvl.readthedocs.io/en/stable/_modules/pvl/decoder.html#PVLDecoder.decode_unquoted_string
- Modification entails removal of decode_datetime call
- """
- for coll in (
- ("a comment", chain.from_iterable(self.grammar.comments)),
- ("some whitespace", self.grammar.whitespace),
- ("a special character", self.grammar.reserved_characters),
- ):
- for item in coll[1]:
- if item in value:
- raise ValueError(
- "Expected a Simple Value, but encountered "
- f'{coll[0]} in "{self}": "{item}".'
- )
-
- agg_keywords = self.grammar.aggregation_keywords.items()
- for kw in chain.from_iterable(agg_keywords):
- if kw.casefold() == value.casefold():
- raise ValueError(
- "Expected a Simple Value, but encountered "
- f'an aggregation keyword: "{value}".'
- )
-
- for es in self.grammar.end_statements:
- if es.casefold() == value.casefold():
- raise ValueError(
- "Expected a Simple Value, but encountered "
- f'an End-Statement: "{value}".'
- )
-
- return str(value)
diff --git a/plio/io/tests/test_io_bae.py b/plio/io/tests/test_io_bae.py
index 4031703..11bffd9 100644
--- a/plio/io/tests/test_io_bae.py
+++ b/plio/io/tests/test_io_bae.py
@@ -3,7 +3,7 @@ import os
import numpy as np
import pandas as pd
-from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
from plio.io.io_bae import socetset_keywords_to_dict, read_gpf, save_gpf, read_ipf, save_ipf
from plio.examples import get_path
diff --git a/plio/io/tests/test_io_gdal.py b/plio/io/tests/test_io_gdal.py
index 0423ca3..a8bc23d 100644
--- a/plio/io/tests/test_io_gdal.py
+++ b/plio/io/tests/test_io_gdal.py
@@ -39,7 +39,7 @@ class TestMercator(unittest.TestCase):
"""
def test_scale(self):
- self.assertEqual(self.dataset.scale, ('Meter', 1.0))
+ self.assertEqual(self.dataset.scale, ('metre', 1.0))
def test_xy_extent(self):
xy_extent = self.dataset.xy_extent
@@ -231,13 +231,17 @@ class TestWriter(unittest.TestCase):
DATUM["Moon_2000",
SPHEROID["Moon_2000_IAU_IAG",1737400,0]],
PRIMEM["Reference_Meridian",0],
- UNIT["Degree",0.017453292519943295]],
- PROJECTION["Mercator_2SP"],
+ UNIT["degree",0.0174532925199433,
+ AUTHORITY["EPSG","9122"]]],
+ PROJECTION["Mercator_1SP"],
PARAMETER["central_meridian",180],
+ PARAMETER["scale_factor",1],
PARAMETER["false_easting",0],
PARAMETER["false_northing",0],
- PARAMETER["standard_parallel_1",0],
- UNIT["Meter",1]]"""
+ UNIT["metre",1,
+ AUTHORITY["EPSG","9001"]],
+ AXIS["Easting",EAST],
+ AXIS["Northing",NORTH]]"""
dataset = io_gdal.GeoDataset('test.tif')
test_srs = dataset.spatial_reference.__str__()
self.assertEqual(test_srs.split(), expected_srs.split())
diff --git a/plio/io/tests/test_metadata.py b/plio/io/tests/test_metadata.py
index 6eef483..08d41a6 100644
--- a/plio/io/tests/test_metadata.py
+++ b/plio/io/tests/test_metadata.py
@@ -1,7 +1,7 @@
import pytest
from plio.io import extract_metadata as em
-from plio.io import gdal
+from plio.io import gdal
@pytest.fixture
def wkt_moon():
@@ -52,7 +52,7 @@ def test_export_to_proj4(srs_mars):
Check that proj4 is not supporting Moon2000_Mercator
"""
proj4 = srs_mars.ExportToProj4()
- for element in '+proj=merc +lon_0=180 +lat_ts=0 +x_0=0 +y_0=0 +a=1737400 +b=1737400 +units=m +no_defs'.split():
+ for element in '+proj=merc +lon_0=180 +lat_ts=0 +x_0=0 +y_0=0 +R=1737400 +units=m +no_defs'.split():
assert element in proj4
@pytest.mark.skipif(gdal is None, reason="GDAL not installed")
--
GitLab