diff --git a/.travis.yml b/.travis.yml
index 027346f8311408e148fbbcf3870bb0ed67090061..9fce481a544bacae0bdfe49eaf0ff598e9d5bce4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,6 +43,7 @@ install:
# Install not using env because build needs to be in root env
- conda config --add channels conda-forge
- conda config --add channels jlaura
+ - conda install python=$PYTHON_VERSION
- conda install -c conda-forge gdal h5py
- conda install pandas sqlalchemy pyyaml networkx
- conda install -c jlaura pvl protobuf
diff --git a/README.rst b/README.rst
index b20dfd71fd8f3077ee035b7b8fe138e6f5770549..c15a55c5faa782be65cce5c594ccf38aff2ddfdc 100644
--- a/README.rst
+++ b/README.rst
@@ -28,12 +28,12 @@ Installation is unfortunately complicated due to the difficulty in installing GD
1. Install Anaconda or Miniconda with Python 3.5
2. In your ~/.condarc file add the following two lines in the channels section.
-
+
`- conda-forge`
-
+
`- jlaura`
-3. Create a new environment to ensure that the installed packge is not going to collide with existing packages
-
- ` conda create --name <somename> python=3`
-
-3. Install `plio` with `conda install plio`.
+3. Create a new environment to ensure that the installed package is not going to collide with existing packages
+
+ ``conda create --name <somename> python=3``
+
+3. Install ``plio`` with ``conda install plio``.
diff --git a/appveyor.yml b/appveyor.yml
index 47dedfaf6f75f9393f5a874551481630c1d5e354..58fa4f2ff020fce49d20478182eb8fd5aa0e33d0 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -40,6 +40,7 @@ install:
- cmd: set PYTHONUNBUFFERED=1
- cmd: conda config --set show_channel_urls true
+ - cmd: conda install --yes python=3.5
- cmd: conda install -c pelson/channel/development --yes --quiet obvious-ci
- cmd: conda config --add channels conda-forge
- cmd: conda info
diff --git a/plio/examples/Tes/pos.fmt b/plio/examples/Tes/pos.fmt
new file mode 100755
index 0000000000000000000000000000000000000000..6a50148f13522e3f933f9c1f966630138c396568
--- /dev/null
+++ b/plio/examples/Tes/pos.fmt
@@ -0,0 +1,87 @@
+NAME = POS
+COLUMNS = 6
+ROW_BYTES = 54
+INTERCHANGE_FORMAT = BINARY
+PRIMARY_KEY = ( "SPACECRAFT_CLOCK_START_COUNT" )
+DESCRIPTION = "
+
+ The POS table stores the positions of the spacecraft and sun
+ relative to the planet, the spacecraft's orientation quaternion,
+ and the Mars body quaternion, all relative to the J2000 system.
+
+ These data are initially derived from the project's SPICE kernels,
+ but may be corrected from various other sources. This table
+ may also include interpolated values where SPICE data were unavailable."
+
+OBJECT = COLUMN
+ NAME = SPACECRAFT_CLOCK_START_COUNT
+ DATA_TYPE = MSB_UNSIGNED_INTEGER
+ START_BYTE = 1
+ BYTES = 4
+ ALIAS_NAME = sclk_time
+ DESCRIPTION = "The value of the spacecraft clock at the
+ beginning of the observation"
+END_OBJECT = COLUMN
+
+
+OBJECT = COLUMN
+ NAME = EPHEMERIS_TIME
+ DATA_TYPE = IEEE_REAL
+ START_BYTE = 5
+ BYTES = 8
+ ALIAS_NAME = et
+ DESCRIPTION = "Ephemeris time, seconds since 1/1/2000"
+ UNIT = "Seconds"
+END_OBJECT = COLUMN
+
+OBJECT = COLUMN
+ NAME = SPACECRAFT_POSITION
+ DATA_TYPE = IEEE_REAL
+ START_BYTE = 13
+ BYTES = 12
+ ITEMS = 3
+ ITEM_BYTES = 4
+ ALIAS_NAME = pos
+ DESCRIPTION = "Spacecraft position vector relative to Mars
+ in the J2000 reference frame"
+ UNIT = "KM"
+END_OBJECT = COLUMN
+
+OBJECT = COLUMN
+ NAME = SUN_POSITION
+ DATA_TYPE = IEEE_REAL
+ START_BYTE = 25
+ BYTES = 12
+ ITEMS = 3
+ ITEM_BYTES = 4
+ ALIAS_NAME = sun
+ DESCRIPTION = "Sun position vector relative to Mars in the J2000
+ reference frame"
+ UNIT = "KM"
+END_OBJECT = COLUMN
+
+OBJECT = COLUMN
+ NAME = SPACECRAFT_QUATERNION
+ DATA_TYPE = IEEE_REAL
+ START_BYTE = 37
+ BYTES = 16
+ ITEMS = 4
+ ITEM_BYTES = 4
+ ALIAS_NAME = quat
+ DESCRIPTION = "Spacecraft pointing quaternion in the J2000
+ reference frame"
+END_OBJECT = COLUMN
+
+OBJECT = COLUMN
+ NAME = POSITION_SOURCE_ID
+ DATA_TYPE = CHARACTER
+ START_BYTE = 53
+ BYTES = 2
+ ITEMS = 2
+ ITEM_BYTES = 1
+ ALIAS_NAME = id
+ DESCRIPTION = "2-character source ID.
+ First character is source of positions.
+ Second character is source of pointing.
+ See ancillary table for details."
+END_OBJECT = COLUMN
diff --git a/plio/examples/Tes/pos10001.tab b/plio/examples/Tes/pos10001.tab
new file mode 100755
index 0000000000000000000000000000000000000000..04ea14e022940553d566ca2ac4ff8e1c5b5d2ba2
Binary files /dev/null and b/plio/examples/Tes/pos10001.tab differ
diff --git a/plio/io/io_tes.py b/plio/io/io_tes.py
new file mode 100644
index 0000000000000000000000000000000000000000..66a6cd141d77ca7c281168e0fab7dd6b09d97c83
--- /dev/null
+++ b/plio/io/io_tes.py
@@ -0,0 +1,246 @@
+import numpy as np
+import pandas as pd
+import pvl
+
+import sys
+import functools
+import json
+
+from plio.io.io_json import read_json
+
+from plio.utils._tes2numpy import tes_dtype_map
+from plio.utils._tes2numpy import tes_columns
+from plio.utils._tes2numpy import tes_scaling_factors
+
+class Tes(object):
+ """
+ Attributes
+ ----------
+
+ spectra : panel
+ A pandas panel containing n individual spectra.
+
+ ancillary_data : dataframe
+ A pandas DataFrame of the parsed ancillary data (PVL label)
+
+ label : object
+ The raw PVL label object
+
+ """
+
+
+ def __init__(self, input_data, var_file = None):
+ """
+ Read the .spc file, parse the label, and extract the spectra
+
+ Parameters
+ ----------
+
+ input_data : string
+ The PATH to the input .tab file
+
+ """
+ def expand_column(df, expand_column, columns): # pragma: no cover
+ array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
+ new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
+ del new_df[expand_column]
+ return new_df
+
+ def bolquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
+
+ types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def obsquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
+ bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
+ bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
+
+ types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
+ ('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def obsclass2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
+ bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
+ bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
+ bitarr2int(bitarr[16:]))]
+
+ types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
+ ('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def radquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
+ bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
+ bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
+ bitarr2int(bitarr[8:9]))]
+
+ types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
+ ('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
+ ('DETECTOR_MASK_PROBLEM', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def atmquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
+
+ types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def expand_column(df, expand_column, columns): # pragma: no cover
+ array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
+ new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
+ del new_df[expand_column]
+ return new_df
+
+ def bolquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
+
+ types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def obsquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
+ bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
+ bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
+
+ types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
+ ('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def obsclass2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
+ bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
+ bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
+ bitarr2int(bitarr[16:]))]
+
+ types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
+ ('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def radquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
+ bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
+ bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
+ bitarr2int(bitarr[8:9]))]
+
+ types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
+ ('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
+ ('DETECTOR_MASK_PROBLEM', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def atmquality2arr(arr): # pragma: no cover
+ bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
+ lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
+
+ types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
+ arr = np.array(lis, dtype=types)
+ return arr
+
+ def bitarr2int(arr): # pragma: no cover
+ arr = "".join(str(i) for i in arr)
+ return np.uint8(int(arr,2))
+
+ def bit2bool(bit): # pragma: no cover
+ return np.bool_(bit)
+
+ def expand_bitstrings(df, dataset): # pragma: no cover
+ if dataset == 'BOL':
+ quality_columns = ['ti_bol_rating', 'bol_ref_lamp']
+ df['quality'] = df['quality'].apply(bolquality2arr)
+ return expand_column(df, 'quality', quality_columns)
+
+ elif dataset == 'OBS':
+ quality_columns = ['hga_motion', 'pnl_motion', 'algor_patch', 'imc_patch',
+ 'momentum', 'equal_tab']
+ class_columns = ['phase', 'type', 'sequence',
+ 'lamp_status', 'timing', 'spare', 'class_value']
+
+ df['quality'] = df['quality'].apply(obsquality2arr)
+ df['class'] = df['class'].apply(obsclass2arr)
+
+ new_df = expand_column(df, 'quality', quality_columns)
+ new_df = expand_column(new_df, 'class', class_columns)
+ return new_df
+
+ elif dataset == 'RAD':
+ quality_columns = ['phase_inversion', 'algor_risk', 'calib_fail', 'calib_quality',
+ 'spect_noise', 'ti_spc_rating', 'det_mask_problem']
+
+ df['quality'] = df['quality'].apply(radquality2arr)
+
+ return expand_column(df, 'quality', quality_columns)
+
+ elif dataset == 'ATM':
+ quality_columns = ['atm_pt_rating', 'atm_opacity_rating']
+
+ df['quality'] = df['quality'].apply(atmquality2arr)
+ return expand_column(df, 'quality', quality_columns)
+
+ else:
+ return df
+
+ self.label = pvl.load(input_data)
+ nrecords = self.label['TABLE']['ROWS']
+ nbytes_per_rec = self.label['RECORD_BYTES']
+ data_start = self.label['LABEL_RECORDS'] * self.label['RECORD_BYTES']
+ dataset = self.label['TABLE']['^STRUCTURE'].split('.')[0]
+
+ numpy_dtypes = tes_dtype_map
+ columns = tes_columns
+ scaling_factors = tes_scaling_factors
+
+ with open(input_data, 'rb') as file:
+ file.seek(data_start)
+ buffer = file.read(nrecords*nbytes_per_rec)
+ array = np.frombuffer(buffer, dtype=numpy_dtypes[dataset.upper()]).byteswap().newbyteorder()
+
+ df = pd.DataFrame(data=array, columns=columns[dataset.upper()])
+
+ # Read Radiance array if applicable
+ if dataset.upper() == 'RAD': # pragma: no cover
+ with open('{}.var'.format(path.splitext(f)[0]) , 'rb') as file:
+ buffer = file.read()
+ def process_rad(index):
+ if index is -1:
+ return None
+
+ length = np.frombuffer(buffer[index:index+2], dtype='>u2')[0]
+ exp = np.frombuffer(buffer[index+2:index+4], dtype='>i2')[0]
+
+ radarr = np.frombuffer(buffer[index+4:index+4+length-2], dtype='>i2') * (2**(exp-15))
+ if np.frombuffer(buffer[index+4+length-2:index+4+length], dtype='>u2')[0] != length:
+ warnings.warn("Last element did not match the length for file index {} in file {}".format(index, f))
+ return radarr
+
+ df["raw_rad"] = df["raw_rad"].apply(process_rad)
+ df["cal_rad"] = df["cal_rad"].apply(process_rad)
+
+ # Apply scaling factors
+ for column in scaling_factors[dataset]: # pragma: no cover
+ def scale(x):
+ return np.multiply(x, scaling_factors[dataset][column])
+ df[column] = df[column].apply(scale)
+
+ df = expand_bitstrings(df, dataset.upper())
+
+ self.data = df
diff --git a/plio/io/tests/test_io_spectral_profiler.py b/plio/io/tests/test_io_spectral_profiler.py
index 3d05e05c4241e3fbc900aa9579072238a3bb06d6..772712de822305bc48f466675b3ee3f4c1d7208e 100644
--- a/plio/io/tests/test_io_spectral_profiler.py
+++ b/plio/io/tests/test_io_spectral_profiler.py
@@ -11,10 +11,10 @@ from plio.io import io_spectral_profiler
from plio.io.io_gdal import GeoDataset
class Test_Spectral_Profiler_IO(unittest.TestCase):
-
+
def setUp(self):
self.examplefile = get_path('SP_2C_02_02358_S138_E3586.spc')
-
+
def test_openspc(self):
ds = io_spectral_profiler.Spectral_Profiler(self.examplefile)
self.assertEqual(ds.nspectra, 38)
diff --git a/plio/io/tests/test_io_tes.py b/plio/io/tests/test_io_tes.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fa5d8d3cde9f673c2a069f8b0206355d27754eb
--- /dev/null
+++ b/plio/io/tests/test_io_tes.py
@@ -0,0 +1,34 @@
+import os
+import sys
+import unittest
+
+import pandas as pd
+
+sys.path.insert(0, os.path.abspath('..'))
+
+from plio.examples import get_path
+from plio.io import io_tes
+from plio.io.io_gdal import GeoDataset
+from plio.utils._tes2numpy import tes2numpy
+
+
+class Test_Tes_IO(unittest.TestCase):
+
+ def setUp(self):
+ self.examplefile = get_path('pos10001.tab')
+
+ def test_opentab(self):
+ ds = io_tes.Tes(self.examplefile)
+ self.assertEqual(ds.data.size, 119106)
+ self.assertIsInstance(ds.data, pd.DataFrame)
+ self.assertEqual(ds.data.columns.tolist(), ['sclk_time', 'et', 'pos', 'sun', 'quat', 'id'])
+
+ def test_tes2numpy(self):
+ self.assertEqual(tes2numpy('MSB_UNSIGNED_INTEGER', 4), '>u4')
+ self.assertEqual(tes2numpy('MSB_UNSIGNED_INTEGER', 4, 2), [('elem0', '>u4'), ('elem1', '>u4')])
+
+ with self.assertRaises(Exception):
+ tes2numpy('IEEE_REAL', 5)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/plio/utils/_tes2numpy.py b/plio/utils/_tes2numpy.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb74ab80607ee9c414f7b4aaca8ed46687d110fc
--- /dev/null
+++ b/plio/utils/_tes2numpy.py
@@ -0,0 +1,405 @@
+def tes2numpy(msb_type, num_bytes, nelems=1):
+ """
+ Converts a MSB data type to a numpy datatype
+
+ """
+ valid_bytes = {
+ 'MSB_UNSIGNED_INTEGER': [1,2,4,8,16,32,64],
+ 'MSB_INTEGER': [1,2,4,8,16,32,64],
+ 'IEEE_REAL': [1,2,4,8,16,32,64],
+ 'CHARACTER': range(1,128),
+ 'MSB_BIT_STRING': range(1,128)
+ }
+
+ msb_bit_string_type = [('byte{}'.format(i), '>u1') for i in range(num_bytes)]
+
+ dtype_map = {
+ 'MSB_UNSIGNED_INTEGER': '>u{}'.format(num_bytes),
+ 'MSB_INTEGER': '>i{}'.format(num_bytes),
+ 'IEEE_REAL': '>f{}'.format(num_bytes),
+ 'CHARACTER': 'a{}'.format(num_bytes),
+ 'MSB_BIT_STRING': msb_bit_string_type
+ }
+
+ if num_bytes not in valid_bytes[msb_type] and nelems == 1:
+ raise Exception('invalid byte ({}) count for type ({})'.format(num_bytes, msb_type))
+
+ if nelems > 1:
+ # Must be an array
+ return [('elem{}'.format(i), dtype_map[msb_type]) for i in range(nelems)]
+
+
+ return dtype_map[msb_type]
+
+
+tes_dtype_map = {'ATM': [('sclk_time', '>u4'),
+ ('srf_pressure', '>u2'),
+ ('nadir_pt',
+ [('elem0', '>u2'),
+ ('elem1', '>u2'),
+ ('elem2', '>u2'),
+ ('elem3', '>u2'),
+ ('elem4', '>u2'),
+ ('elem5', '>u2'),
+ ('elem6', '>u2'),
+ ('elem7', '>u2'),
+ ('elem8', '>u2'),
+ ('elem9', '>u2'),
+ ('elem10', '>u2'),
+ ('elem11', '>u2'),
+ ('elem12', '>u2'),
+ ('elem13', '>u2'),
+ ('elem14', '>u2'),
+ ('elem15', '>u2'),
+ ('elem16', '>u2'),
+ ('elem17', '>u2'),
+ ('elem18', '>u2'),
+ ('elem19', '>u2'),
+ ('elem20', '>u2'),
+ ('elem21', '>u2'),
+ ('elem22', '>u2'),
+ ('elem23', '>u2'),
+ ('elem24', '>u2'),
+ ('elem25', '>u2'),
+ ('elem26', '>u2'),
+ ('elem27', '>u2'),
+ ('elem28', '>u2'),
+ ('elem29', '>u2'),
+ ('elem30', '>u2'),
+ ('elem31', '>u2'),
+ ('elem32', '>u2'),
+ ('elem33', '>u2'),
+ ('elem34', '>u2'),
+ ('elem35', '>u2'),
+ ('elem36', '>u2'),
+ ('elem37', '>u2')]),
+ ('co2_cont_temp', '>u2'),
+ ('srf_temp_est', '>u2'),
+ ('rms_pt', '>f4'),
+ ('best_fit_opacities',
+ [('elem0', '>i2'),
+ ('elem1', '>i2'),
+ ('elem2', '>i2'),
+ ('elem3', '>i2'),
+ ('elem4', '>i2'),
+ ('elem5', '>i2'),
+ ('elem6', '>i2'),
+ ('elem7', '>i2'),
+ ('elem8', '>i2')]),
+ ('rms_opacities', '>f4'),
+ ('co2_dw_flux', '>f4'),
+ ('total_dw_flux', '>f4'),
+ ('quality', [('byte0', '>u1'), ('byte1', '>u1')]),
+ ('srf_radiance', '>i4'),
+ ('version_id', 'a4')],
+ 'BOL': [('sclk_time', '>u4'),
+ ('detector', '>u1'),
+ ('tic_count', '>u1'),
+ ('vbol', '>i2'),
+ ('tbol', '>i2'),
+ ('cal_vbol', '>f4'),
+ ('lambert_alb', '>f4'),
+ ('ti_bol', '>f4'),
+ ('brightness_temp_bol', '>u2'),
+ ('vbol_version_id', 'a2'),
+ ('tbol_version_id', 'a2'),
+ ('quality', [('byte0', '>u1'), ('byte1', '>u1')])],
+ 'GEO': [('sclk_time', '>u4'),
+ ('detector', '>u1'),
+ ('longitude', '>u2'),
+ ('latitude', '>i2'),
+ ('phase', '>u2'),
+ ('emission', '>u2'),
+ ('incidence', '>u2'),
+ ('planetary_phase', '>u2'),
+ ('heliocentric_lon', '>u2'),
+ ('sub_sc_lon', '>u2'),
+ ('sub_sc_lat', '>i2'),
+ ('sub_solar_lon', '>u2'),
+ ('sub_solar_lat', '>i2'),
+ ('target_distance', '>u2'),
+ ('height', '>u2'),
+ ('altitude', '>u2'),
+ ('local_time', '>u2'),
+ ('solar_distance', '>u2'),
+ ('angular_semidiameter', '>u2'),
+ ('version_id', 'a4')],
+ 'IFG': [('sclk_time', '>u4'), ('detector', '>u1'), ('ifgm', '>i4')],
+ 'OBS': [('sclk_time', '>u4'),
+ ('orbit', '>u2'),
+ ('ock', '>u2'),
+ ('ick', '>u4'),
+ ('tic', '>u1'),
+ ('pnt_angle', '>i2'),
+ ('pnt_imc', '>u1'),
+ ('pnt_view', 'a1'),
+ ('scan_len', 'a1'),
+ ('pckt_type', 'a1'),
+ ('schedule_type', 'a1'),
+ ('spc_gain', 'a1'),
+ ('vbol_gain', 'a1'),
+ ('tbol_gain', 'a1'),
+ ('comp_pp', '>u1'),
+ ('det_mask', '>u1'),
+ ('class',
+ [('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')]),
+ ('quality',
+ [('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')]),
+ ('temps',
+ [('elem0', '>u2'), ('elem1', '>u2'), ('elem2', '>u2'), ('elem3', '>u2')]),
+ ('ffti', '>u1')],
+ 'POS': [('sclk_time', '>u4'),
+ ('et', '>f8'),
+ ('pos', [('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4')]),
+ ('sun', [('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4')]),
+ ('quat',
+ [('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4'), ('elem3', '>f4')]),
+ ('id', [('elem0', 'a1'), ('elem1', 'a1')])],
+ 'RAD': [('sclk_time', '>u4'),
+ ('detector', '>u1'),
+ ('spectral_mask', '>u1'),
+ ('cmode', '>u2'),
+ ('raw_rad', '>i4'),
+ ('cal_rad', '>i4'),
+ ('tdet', '>u2'),
+ ('target_temp', '>u2'),
+ ('ti_spc', '>f4'),
+ ('version_id', 'a4'),
+ ('quality',
+ [('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')])],
+ 'TLM': [('sclk_time', '>u4'),
+ ('aux_temps',
+ [('elem0', '>u2'),
+ ('elem1', '>u2'),
+ ('elem2', '>u2'),
+ ('elem3', '>u2'),
+ ('elem4', '>u2'),
+ ('elem5', '>u2'),
+ ('elem6', '>u2'),
+ ('elem7', '>u2'),
+ ('elem8', '>u2'),
+ ('elem9', '>u2'),
+ ('elem10', '>u2'),
+ ('elem11', '>u2')]),
+ ('ifgm_max',
+ [('elem0', '>i2'),
+ ('elem1', '>i2'),
+ ('elem2', '>i2'),
+ ('elem3', '>i2'),
+ ('elem4', '>i2'),
+ ('elem5', '>i2')]),
+ ('ifgm_min',
+ [('elem0', '>i2'),
+ ('elem1', '>i2'),
+ ('elem2', '>i2'),
+ ('elem3', '>i2'),
+ ('elem4', '>i2'),
+ ('elem5', '>i2')]),
+ ('dsp_log',
+ [('elem0', '>u2'),
+ ('elem1', '>u2'),
+ ('elem2', '>u2'),
+ ('elem3', '>u2'),
+ ('elem4', '>u2'),
+ ('elem5', '>u2')]),
+ ('V1', '>i1'),
+ ('V2', '>i1'),
+ ('V3', '>i1'),
+ ('V4', '>i1'),
+ ('V5', '>i1'),
+ ('V6', '>i1'),
+ ('V7', '>i1'),
+ ('V8', '>i1'),
+ ('V9', '>i1'),
+ ('V10', '>i1'),
+ ('V11', '>i1'),
+ ('V12', '>i1'),
+ ('V13', '>i1'),
+ ('V14', '>i1'),
+ ('V15', '>i1'),
+ ('V16', '>i1'),
+ ('V17', '>i1'),
+ ('V18', '>i1'),
+ ('V19', '>i1'),
+ ('V20', '>i1'),
+ ('neon_lamp', '>u1'),
+ ('neon_gain', 'a1'),
+ ('neon_amp', '>i1'),
+ ('neon_zpd', '>u2'),
+ ('ifgm_zpd',
+ [('elem0', '>u2'),
+ ('elem1', '>u2'),
+ ('elem2', '>u2'),
+ ('elem3', '>u2'),
+ ('elem4', '>u2'),
+ ('elem5', '>u2')]),
+ ('ifgm_end',
+ [('elem0', '>u2'),
+ ('elem1', '>u2'),
+ ('elem2', '>u2'),
+ ('elem3', '>u2'),
+ ('elem4', '>u2'),
+ ('elem5', '>u2')])]}
+
+tes_columns = {'ATM': ['sclk_time',
+ 'srf_pressure',
+ 'nadir_pt',
+ 'co2_cont_temp',
+ 'srf_temp_est',
+ 'rms_pt',
+ 'best_fit_opacities',
+ 'rms_opacities',
+ 'co2_dw_flux',
+ 'total_dw_flux',
+ 'quality',
+ 'srf_radiance',
+ 'version_id'],
+ 'BOL': ['sclk_time',
+ 'detector',
+ 'tic_count',
+ 'vbol',
+ 'tbol',
+ 'cal_vbol',
+ 'lambert_alb',
+ 'ti_bol',
+ 'brightness_temp_bol',
+ 'vbol_version_id',
+ 'tbol_version_id',
+ 'quality'],
+ 'GEO': ['sclk_time',
+ 'detector',
+ 'longitude',
+ 'latitude',
+ 'phase',
+ 'emission',
+ 'incidence',
+ 'planetary_phase',
+ 'heliocentric_lon',
+ 'sub_sc_lon',
+ 'sub_sc_lat',
+ 'sub_solar_lon',
+ 'sub_solar_lat',
+ 'target_distance',
+ 'height',
+ 'altitude',
+ 'local_time',
+ 'solar_distance',
+ 'angular_semidiameter',
+ 'version_id'],
+ 'IFG': ['sclk_time', 'detector', 'ifgm'],
+ 'OBS': ['sclk_time',
+ 'orbit',
+ 'ock',
+ 'ick',
+ 'tic',
+ 'pnt_angle',
+ 'pnt_imc',
+ 'pnt_view',
+ 'scan_len',
+ 'pckt_type',
+ 'schedule_type',
+ 'spc_gain',
+ 'vbol_gain',
+ 'tbol_gain',
+ 'comp_pp',
+ 'det_mask',
+ 'class',
+ 'quality',
+ 'temps',
+ 'ffti'],
+ 'POS': ['sclk_time', 'et', 'pos', 'sun', 'quat', 'id'],
+ 'RAD': ['sclk_time',
+ 'detector',
+ 'spectral_mask',
+ 'cmode',
+ 'raw_rad',
+ 'cal_rad',
+ 'tdet',
+ 'target_temp',
+ 'ti_spc',
+ 'version_id',
+ 'quality'],
+ 'TLM': ['sclk_time',
+ 'aux_temps',
+ 'ifgm_max',
+ 'ifgm_min',
+ 'dsp_log',
+ 'V1',
+ 'V2',
+ 'V3',
+ 'V4',
+ 'V5',
+ 'V6',
+ 'V7',
+ 'V8',
+ 'V9',
+ 'V10',
+ 'V11',
+ 'V12',
+ 'V13',
+ 'V14',
+ 'V15',
+ 'V16',
+ 'V17',
+ 'V18',
+ 'V19',
+ 'V20',
+ 'neon_lamp',
+ 'neon_gain',
+ 'neon_amp',
+ 'neon_zpd',
+ 'ifgm_zpd',
+ 'ifgm_end']}
+
+tes_scaling_factors = {'ATM': {'best_fit_opacities': 0.001,
+ 'co2_cont_temp': 0.01,
+ 'nadir_pt': 0.01,
+ 'srf_pressure': 0.001,
+ 'srf_temp_est': 0.01},
+ 'BOL': {'brightness_temp_bol': 0.01,
+ 'tbol': 0.000152587890625,
+ 'vbol': 0.000152587890625},
+ 'CMP': {},
+ 'GEO': {'angular_semidiameter': 0.01,
+ 'emission': 0.01,
+ 'height': 0.01,
+ 'heliocentric_lon': 0.01,
+ 'incidence': 0.01,
+ 'latitude': 0.01,
+ 'local_time': 0.001,
+ 'longitude': 0.01,
+ 'phase': 0.01,
+ 'planetary_phase': 0.01,
+ 'solar_distance': 10000,
+ 'sub_sc_lat': 0.01,
+ 'sub_sc_lon': 0.01,
+ 'sub_solar_lat': 0.01,
+ 'sub_solar_lon': 0.01},
+ 'IFG': {},
+ 'OBS': {'pnt_angle': 0.046875, 'temps': 0.01},
+ 'PCT': {},
+ 'POS': {},
+ 'RAD': {'target_temp': 0.01, 'tdet': 0.01},
+ 'TLM': {'V1': 3.90625,
+ 'V10': -0.15625,
+ 'V11': 0.0976055,
+ 'V12': -0.0985813,
+ 'V13': 0.976562,
+ 'V14': 0.0648437,
+ 'V15': 0.045727,
+ 'V16': 0.0480992,
+ 'V17': 0.0478277,
+ 'V18': 0.0488039,
+ 'V19': 0.141966,
+ 'V2': 1.95312,
+ 'V20': -0.149688,
+ 'V3': 0.278906,
+ 'V4': 0.278906,
+ 'V5': 4.45312,
+ 'V6': 0.652344,
+ 'V7': 0.119457,
+ 'V8': -0.103067,
+ 'V9': 0.15576,
+ 'aux_temps': 0.01,
+ 'ifgm_max': 0.000152587890625,
+ 'ifgm_min': 0.000152587890625}}