diff --git a/plio/examples/SpectralProfiler/cl5_398736801edr_f0030004ccam01014m1.dat b/plio/examples/SpectralProfiler/cl5_398736801edr_f0030004ccam01014m1.dat
new file mode 100644
index 0000000000000000000000000000000000000000..b9dcce79d7d708162dfe49db82fcb094208ef3b5
Binary files /dev/null and b/plio/examples/SpectralProfiler/cl5_398736801edr_f0030004ccam01014m1.dat differ
diff --git a/plio/examples/__init__.py b/plio/examples/__init__.py
index 405e9c68f4fc296fa3748d681b97f9bbd1ce6eea..68f159dfefaa1a775abba91ce5c61128ca9b448b 100644
--- a/plio/examples/__init__.py
+++ b/plio/examples/__init__.py
@@ -3,7 +3,7 @@ import plio
__all__ = ['available', 'get_path']
-#Used largely unmodififed from:
+# Used largely unmodififed from:
# https://github.com/pysal/pysal/blob/master/pysal/examples/__init__.py
base = os.path.split(plio.__file__)[0]
diff --git a/plio/io/io_edr.py b/plio/io/io_edr.py
new file mode 100644
index 0000000000000000000000000000000000000000..caa9d6d396b5917b04c22e22632f93c3a775112d
--- /dev/null
+++ b/plio/io/io_edr.py
@@ -0,0 +1,80 @@
+import os
+
+import numpy as np
+import pandas as pd
+
+
+def EDR(input_file):
+ f = open(input_file, 'rb') # read as bytes so python won't complain about the binary part of the file
+
+ # read lines of the header until reaching the end of the libs table (collecting other metadata along the way)
+ end_of_libs_table = False
+ while end_of_libs_table is False:
+ line = str(f.readline(), 'utf-8').replace('\r', '').replace('\n',
+ '') # convert the current line to a string and get rid of newline characters
+ line = line.split('=') # split the line on equals sign if present
+ # look for the name of the value we want, if the current line has it, then set the value
+ if 'RECORD_BYTES' in line[0]:
+ rbytes = int(line[1])
+ if 'LABEL_RECORDS' in line[0]:
+ lrecs = int(line[1])
+ if 'SPACECRAFT_CLOCK_START_COUNT' in line[0]:
+ sclock = int(line[1].replace('"', '').split('.')[0])
+ if 'SEQUENCE_ID' in line[0]:
+ seqID = line[1].replace('"', '')
+ if 'INSTRUMENT_FOCUS_DISTANCE' in line[0]:
+ focus_dist = int(line[1])
+
+ if 'INSTRUMENT_TEMPERATURE' in line[0]:
+ instrument_temps = line[1] \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '') \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '') \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '')
+ instrument_temps = [float(i) for i in
+ instrument_temps.replace('<degC>', '').replace('(', '').replace(')', '').replace(' ',
+ '').split(
+ ',')]
+ instrument_temps_name = str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '')
+ instrument_temps_name = instrument_temps_name.split('=')[1] \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '') \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '') \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '') \
+ + str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '')
+ instrument_temps_name = instrument_temps_name.replace(' ', '').replace('(', '').replace(')', '').replace(
+ '"', '').split(',')
+ f.readline()
+ pass
+ try:
+ if 'CCAM_LIBS_DATA_CONTAINER' in line[1]:
+ nshots = int(str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '').split('=')[1])
+ start_byte = int(str(f.readline(), 'utf-8').replace('\r', '').replace('\n', '').split('=')[1])
+ if 'END_OBJECT' in line[0] and 'CCAM_LIBS_TABLE' in line[1]:
+ end_of_libs_table = True
+ except:
+ pass
+
+ f.close()
+ header_skip = lrecs * rbytes # calculate the number of header bytes to skip to get to the real data
+
+ with open(input_file, "rb") as f:
+ f.seek(header_skip + start_byte - 1, 0)
+ spectra = []
+ while spectra.__len__() < nshots:
+ spectrum = []
+ while spectrum.__len__() < 6444:
+ spectrum.append(int.from_bytes(f.read(2), byteorder='big', signed=False))
+ spectra.append(spectrum)
+ spectra = np.array(spectra, dtype='int')
+ cols = np.array(list(range(spectra.shape[1]))) + 1
+ cols = [('channel', i) for i in cols]
+ inds = np.array(list(range(spectra.shape[0]))) + 1
+ sp = pd.DataFrame(spectra, columns=pd.MultiIndex.from_tuples(cols), index=inds)
+ sp[('meta', 'EDR_file')] = os.path.basename(input_file)
+ sp[('meta', 'Spacecraft_Clock')] = sclock
+ sp[('meta', 'Shot')] = sp.index
+ sp[('meta', 'SeqID')] = seqID
+ sp[('meta', 'Focus_Distance')] = focus_dist
+ for ind, name in enumerate(instrument_temps_name):
+ sp[('meta', name + '_temp')] = instrument_temps[ind]
+ sp.to_csv('test.csv')
+ return sp
diff --git a/plio/io/io_moon_minerology_mapper.py b/plio/io/io_moon_minerology_mapper.py
new file mode 100644
index 0000000000000000000000000000000000000000..a479ca5e8f656f0495c907a92daf6597a924214c
--- /dev/null
+++ b/plio/io/io_moon_minerology_mapper.py
@@ -0,0 +1,25 @@
+import numpy as np
+from osgeo import gdal
+
+
+def openm3(input_data):
+ if input_data.split('.')[-1] == 'hdr':
+ # GDAL wants the img, but many users aim at the .hdr
+ input_data = input_data.split('.')[0] + '.img'
+ ds = gdal.Open(input_data)
+ ref_array = ds.GetRasterBand(1).ReadAsArray()
+ metadata = ds.GetMetadata()
+ wv_array = metadatatoband(metadata)
+ return wv_array, ref_array, ds
+
+
+def metadatatoband(metadata):
+ wv2band = []
+ for k, v in metadata.iteritems():
+ try:
+ wv2band.append(float(value))
+ except:
+ v = v.split(" ")[-1].split("(")[1].split(")")[0]
+ wv2band.append(float(v))
+ wv2band.sort(key=int)
+ return np.asarray(wv2band)
diff --git a/plio/io/io_multibandimager.py b/plio/io/io_multibandimager.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e5ba2627a240fdc980c9c25a184d93d22168e80
--- /dev/null
+++ b/plio/io/io_multibandimager.py
@@ -0,0 +1,28 @@
+import numpy as np
+from osgeo import gdal
+
+
+def openmi(input_data):
+ ds = gdal.Open(input_data)
+ band_pointers = []
+ nbands = ds.RasterCount
+
+ for b in xrange(1, nbands + 1):
+ band_pointers.append(ds.GetRasterBand(b))
+
+ ref_array = ds.GetRasterBand(1).ReadAsArray()
+ wv_array = None
+ return wv_array, ref_array[::3, ::3], ds
+
+
+def getspectra(x, y, ds):
+ nbands = ds.RasterCount
+ reflectance = np.empty(nbands)
+ for b in range(1, nbands + 1):
+ reflectance[b - 1] = ds.GetRasterBand(b).ReadAsArray(y, x, 1, 1)
+
+ mergedref = np.empty(nbands - 1)
+ mergedref[:4] = reflectance[:4]
+ mergedref[4] = (reflectance[4] + reflectance[5]) / 2
+ mergedref[5:] = reflectance[6:]
+ return mergedref
diff --git a/plio/io/io_yaml.py b/plio/io/io_yaml.py
index 38c787ae91d697aa1efe20ead1387edc56aeaa41..59d1a70d9d24fc43f8a68b658826b27a81c069fb 100644
--- a/plio/io/io_yaml.py
+++ b/plio/io/io_yaml.py
@@ -1,7 +1,4 @@
-try:
- import yaml
-except:
- print('YAML package not installed, disabling yaml_io module')
+import yaml
def read_yaml(inputfile):
@@ -21,6 +18,6 @@ def read_yaml(inputfile):
try:
with open(inputfile, 'r') as f:
ydict = yaml.load(f)
- except: # pragma: no cover
+ except: # pragma: no cover
raise IOError('Unable to load YAML file.')
return ydict
diff --git a/plio/io/tests/test_io_edr.py b/plio/io/tests/test_io_edr.py
new file mode 100644
index 0000000000000000000000000000000000000000..a321f6eb17c4dc3eeac3914fad5e4b02e3a4b016
--- /dev/null
+++ b/plio/io/tests/test_io_edr.py
@@ -0,0 +1,19 @@
+import unittest
+
+sys.path.insert(0, os.path.abspath('..'))
+
+from plio.examples import get_path
+from plio.io import io_edr
+
+
+class Test_Tes_IO(unittest.TestCase):
+
+ # Need different test data or need to modify the current code
+ def setUp(self):
+ self.examplefile = get_path('cl5_398736801edr_f0030004ccam01014m1.dat')
+
+ def test_open(self):
+ ds = io_edr.EDR(self.examplefile)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/plio/io/tests/test_structured_io.py b/plio/io/tests/test_structured_io.py
index 13a7c4195abfcd2f67cd2816d4b0a641c354befd..84e67b8b59225845c96f98c20766a4c47416cce8 100644
--- a/plio/io/tests/test_structured_io.py
+++ b/plio/io/tests/test_structured_io.py
@@ -1,7 +1,7 @@
import unittest
-from .. import io_json
-from .. import io_yaml
+from plio.io import io_json
+from plio.io import io_yaml
try:
import yaml
diff --git a/plio/utils/utils.py b/plio/utils/utils.py
index 761f70d28416f338b896d10f1daff6c4d7eb3ad2..e499cd8418e67d0efb8262bcfaf9f710cc32e735 100644
--- a/plio/utils/utils.py
+++ b/plio/utils/utils.py
@@ -5,6 +5,7 @@ import os
import fnmatch
import shutil
import tempfile
+import pandas as pd
def create_dir(basedir=''):
@@ -164,3 +165,31 @@ def xstr(s):
if s is None:
return ''
return str(s)
+
+def lookup(df,lookupfile=None,lookupdf=None,sep=',',skiprows=1,left_on='sclock',right_on='Spacecraft Clock'):
+#TODO: automatically determine the number of rows to skip to handle ccam internal master list and PDS "official" master list formats
+ if lookupfile is not None:
+ # this loop concatenates together multiple lookup files if provided
+ # (mostly to handle the three different master lists for chemcam)
+ for x in lookupfile:
+ try:
+ tmp = pd.read_csv(x, sep=sep, skiprows=skiprows, error_bad_lines=False)
+ lookupdf = pd.concat([lookupdf, tmp])
+ except:
+ lookupdf = pd.read_csv(x, sep=sep, skiprows=skiprows, error_bad_lines=False)
+ metadata = df['meta']
+
+ metadata = metadata.merge(lookupdf, left_on=left_on, right_on=right_on, how='left')
+
+ # remove metadata columns that already exist in the data frame to avoid non-unique columns
+ meta_cols = set(metadata.columns.values)
+ meta_cols_keep = list(meta_cols - set(df['meta'].columns.values))
+ metadata = metadata[meta_cols_keep]
+
+ # make metadata into a multiindex
+ metadata.columns = [['meta'] * len(metadata.columns), metadata.columns.values]
+ # give it the same indices as the df
+ metadata.index = df.index
+ # combine the df and the new metadata
+ df = pd.concat([metadata, df], axis=1)
+ return df