Skip to content
Snippets Groups Projects
Commit 0e520231 authored by Kelvin Rodriguez's avatar Kelvin Rodriguez
Browse files

Added Tes data support

parent f59ebf64
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
plio/examples/Tes/pos.fmt 0 → 100755
+ 87
− 0
View file @ 0e520231
NAME = POS
COLUMNS = 6
ROW_BYTES = 54
INTERCHANGE_FORMAT = BINARY
PRIMARY_KEY = ( "SPACECRAFT_CLOCK_START_COUNT" )
DESCRIPTION = "
The POS table stores the positions of the spacecraft and sun
relative to the planet, the spacecraft's orientation quaternion,
and the Mars body quaternion, all relative to the J2000 system.
These data are initially derived from the project's SPICE kernels,
but may be corrected from various other sources. This table
may also include interpolated values where SPICE data were unavailable."
OBJECT = COLUMN
NAME = SPACECRAFT_CLOCK_START_COUNT
DATA_TYPE = MSB_UNSIGNED_INTEGER
START_BYTE = 1
BYTES = 4
ALIAS_NAME = sclk_time
DESCRIPTION = "The value of the spacecraft clock at the
beginning of the observation"
END_OBJECT = COLUMN
OBJECT = COLUMN
NAME = EPHEMERIS_TIME
DATA_TYPE = IEEE_REAL
START_BYTE = 5
BYTES = 8
ALIAS_NAME = et
DESCRIPTION = "Ephemeris time, seconds since 1/1/2000"
UNIT = "Seconds"
END_OBJECT = COLUMN
OBJECT = COLUMN
NAME = SPACECRAFT_POSITION
DATA_TYPE = IEEE_REAL
START_BYTE = 13
BYTES = 12
ITEMS = 3
ITEM_BYTES = 4
ALIAS_NAME = pos
DESCRIPTION = "Spacecraft position vector relative to Mars
in the J2000 reference frame"
UNIT = "KM"
END_OBJECT = COLUMN
OBJECT = COLUMN
NAME = SUN_POSITION
DATA_TYPE = IEEE_REAL
START_BYTE = 25
BYTES = 12
ITEMS = 3
ITEM_BYTES = 4
ALIAS_NAME = sun
DESCRIPTION = "Sun position vector relative to Mars in the J2000
reference frame"
UNIT = "KM"
END_OBJECT = COLUMN
OBJECT = COLUMN
NAME = SPACECRAFT_QUATERNION
DATA_TYPE = IEEE_REAL
START_BYTE = 37
BYTES = 16
ITEMS = 4
ITEM_BYTES = 4
ALIAS_NAME = quat
DESCRIPTION = "Spacecraft pointing quaternion in the J2000
reference frame"
END_OBJECT = COLUMN
OBJECT = COLUMN
NAME = POSITION_SOURCE_ID
DATA_TYPE = CHARACTER
START_BYTE = 53
BYTES = 2
ITEMS = 2
ITEM_BYTES = 1
ALIAS_NAME = id
DESCRIPTION = "2-character source ID.
First character is source of positions.
Second character is source of pointing.
See ancillary table for details."
END_OBJECT = COLUMN
plio/examples/Tes/pos10001.tab 0 → 100755
+ 0
− 0
View file @ 0e520231
File added
plio/io/__init__.py
+ 371
− 0
View file @ 0e520231
tes_dtype_map = {'ATM': [('sclk_time', '>u4'),
('srf_pressure', '>u2'),
('nadir_pt',
[('elem0', '>u2'),
('elem1', '>u2'),
('elem2', '>u2'),
('elem3', '>u2'),
('elem4', '>u2'),
('elem5', '>u2'),
('elem6', '>u2'),
('elem7', '>u2'),
('elem8', '>u2'),
('elem9', '>u2'),
('elem10', '>u2'),
('elem11', '>u2'),
('elem12', '>u2'),
('elem13', '>u2'),
('elem14', '>u2'),
('elem15', '>u2'),
('elem16', '>u2'),
('elem17', '>u2'),
('elem18', '>u2'),
('elem19', '>u2'),
('elem20', '>u2'),
('elem21', '>u2'),
('elem22', '>u2'),
('elem23', '>u2'),
('elem24', '>u2'),
('elem25', '>u2'),
('elem26', '>u2'),
('elem27', '>u2'),
('elem28', '>u2'),
('elem29', '>u2'),
('elem30', '>u2'),
('elem31', '>u2'),
('elem32', '>u2'),
('elem33', '>u2'),
('elem34', '>u2'),
('elem35', '>u2'),
('elem36', '>u2'),
('elem37', '>u2')]),
('co2_cont_temp', '>u2'),
('srf_temp_est', '>u2'),
('rms_pt', '>f4'),
('best_fit_opacities',
[('elem0', '>i2'),
('elem1', '>i2'),
('elem2', '>i2'),
('elem3', '>i2'),
('elem4', '>i2'),
('elem5', '>i2'),
('elem6', '>i2'),
('elem7', '>i2'),
('elem8', '>i2')]),
('rms_opacities', '>f4'),
('co2_dw_flux', '>f4'),
('total_dw_flux', '>f4'),
('quality', [('byte0', '>u1'), ('byte1', '>u1')]),
('srf_radiance', '>i4'),
('version_id', 'a4')],
'BOL': [('sclk_time', '>u4'),
('detector', '>u1'),
('tic_count', '>u1'),
('vbol', '>i2'),
('tbol', '>i2'),
('cal_vbol', '>f4'),
('lambert_alb', '>f4'),
('ti_bol', '>f4'),
('brightness_temp_bol', '>u2'),
('vbol_version_id', 'a2'),
('tbol_version_id', 'a2'),
('quality', [('byte0', '>u1'), ('byte1', '>u1')])],
'GEO': [('sclk_time', '>u4'),
('detector', '>u1'),
('longitude', '>u2'),
('latitude', '>i2'),
('phase', '>u2'),
('emission', '>u2'),
('incidence', '>u2'),
('planetary_phase', '>u2'),
('heliocentric_lon', '>u2'),
('sub_sc_lon', '>u2'),
('sub_sc_lat', '>i2'),
('sub_solar_lon', '>u2'),
('sub_solar_lat', '>i2'),
('target_distance', '>u2'),
('height', '>u2'),
('altitude', '>u2'),
('local_time', '>u2'),
('solar_distance', '>u2'),
('angular_semidiameter', '>u2'),
('version_id', 'a4')],
'IFG': [('sclk_time', '>u4'), ('detector', '>u1'), ('ifgm', '>i4')],
'OBS': [('sclk_time', '>u4'),
('orbit', '>u2'),
('ock', '>u2'),
('ick', '>u4'),
('tic', '>u1'),
('pnt_angle', '>i2'),
('pnt_imc', '>u1'),
('pnt_view', 'a1'),
('scan_len', 'a1'),
('pckt_type', 'a1'),
('schedule_type', 'a1'),
('spc_gain', 'a1'),
('vbol_gain', 'a1'),
('tbol_gain', 'a1'),
('comp_pp', '>u1'),
('det_mask', '>u1'),
('class',
[('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')]),
('quality',
[('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')]),
('temps',
[('elem0', '>u2'), ('elem1', '>u2'), ('elem2', '>u2'), ('elem3', '>u2')]),
('ffti', '>u1')],
'POS': [('sclk_time', '>u4'),
('et', '>f8'),
('pos', [('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4')]),
('sun', [('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4')]),
('quat',
[('elem0', '>f4'), ('elem1', '>f4'), ('elem2', '>f4'), ('elem3', '>f4')]),
('id', [('elem0', 'a1'), ('elem1', 'a1')])],
'RAD': [('sclk_time', '>u4'),
('detector', '>u1'),
('spectral_mask', '>u1'),
('cmode', '>u2'),
('raw_rad', '>i4'),
('cal_rad', '>i4'),
('tdet', '>u2'),
('target_temp', '>u2'),
('ti_spc', '>f4'),
('version_id', 'a4'),
('quality',
[('byte0', '>u1'), ('byte1', '>u1'), ('byte2', '>u1'), ('byte3', '>u1')])],
'TLM': [('sclk_time', '>u4'),
('aux_temps',
[('elem0', '>u2'),
('elem1', '>u2'),
('elem2', '>u2'),
('elem3', '>u2'),
('elem4', '>u2'),
('elem5', '>u2'),
('elem6', '>u2'),
('elem7', '>u2'),
('elem8', '>u2'),
('elem9', '>u2'),
('elem10', '>u2'),
('elem11', '>u2')]),
('ifgm_max',
[('elem0', '>i2'),
('elem1', '>i2'),
('elem2', '>i2'),
('elem3', '>i2'),
('elem4', '>i2'),
('elem5', '>i2')]),
('ifgm_min',
[('elem0', '>i2'),
('elem1', '>i2'),
('elem2', '>i2'),
('elem3', '>i2'),
('elem4', '>i2'),
('elem5', '>i2')]),
('dsp_log',
[('elem0', '>u2'),
('elem1', '>u2'),
('elem2', '>u2'),
('elem3', '>u2'),
('elem4', '>u2'),
('elem5', '>u2')]),
('V1', '>i1'),
('V2', '>i1'),
('V3', '>i1'),
('V4', '>i1'),
('V5', '>i1'),
('V6', '>i1'),
('V7', '>i1'),
('V8', '>i1'),
('V9', '>i1'),
('V10', '>i1'),
('V11', '>i1'),
('V12', '>i1'),
('V13', '>i1'),
('V14', '>i1'),
('V15', '>i1'),
('V16', '>i1'),
('V17', '>i1'),
('V18', '>i1'),
('V19', '>i1'),
('V20', '>i1'),
('neon_lamp', '>u1'),
('neon_gain', 'a1'),
('neon_amp', '>i1'),
('neon_zpd', '>u2'),
('ifgm_zpd',
[('elem0', '>u2'),
('elem1', '>u2'),
('elem2', '>u2'),
('elem3', '>u2'),
('elem4', '>u2'),
('elem5', '>u2')]),
('ifgm_end',
[('elem0', '>u2'),
('elem1', '>u2'),
('elem2', '>u2'),
('elem3', '>u2'),
('elem4', '>u2'),
('elem5', '>u2')])]}
tes_columns = {'ATM': ['sclk_time',
'srf_pressure',
'nadir_pt',
'co2_cont_temp',
'srf_temp_est',
'rms_pt',
'best_fit_opacities',
'rms_opacities',
'co2_dw_flux',
'total_dw_flux',
'quality',
'srf_radiance',
'version_id'],
'BOL': ['sclk_time',
'detector',
'tic_count',
'vbol',
'tbol',
'cal_vbol',
'lambert_alb',
'ti_bol',
'brightness_temp_bol',
'vbol_version_id',
'tbol_version_id',
'quality'],
'GEO': ['sclk_time',
'detector',
'longitude',
'latitude',
'phase',
'emission',
'incidence',
'planetary_phase',
'heliocentric_lon',
'sub_sc_lon',
'sub_sc_lat',
'sub_solar_lon',
'sub_solar_lat',
'target_distance',
'height',
'altitude',
'local_time',
'solar_distance',
'angular_semidiameter',
'version_id'],
'IFG': ['sclk_time', 'detector', 'ifgm'],
'OBS': ['sclk_time',
'orbit',
'ock',
'ick',
'tic',
'pnt_angle',
'pnt_imc',
'pnt_view',
'scan_len',
'pckt_type',
'schedule_type',
'spc_gain',
'vbol_gain',
'tbol_gain',
'comp_pp',
'det_mask',
'class',
'quality',
'temps',
'ffti'],
'POS': ['sclk_time', 'et', 'pos', 'sun', 'quat', 'id'],
'RAD': ['sclk_time',
'detector',
'spectral_mask',
'cmode',
'raw_rad',
'cal_rad',
'tdet',
'target_temp',
'ti_spc',
'version_id',
'quality'],
'TLM': ['sclk_time',
'aux_temps',
'ifgm_max',
'ifgm_min',
'dsp_log',
'V1',
'V2',
'V3',
'V4',
'V5',
'V6',
'V7',
'V8',
'V9',
'V10',
'V11',
'V12',
'V13',
'V14',
'V15',
'V16',
'V17',
'V18',
'V19',
'V20',
'neon_lamp',
'neon_gain',
'neon_amp',
'neon_zpd',
'ifgm_zpd',
'ifgm_end']}
tes_scaling_factors = {'ATM': {'best_fit_opacities': 0.001,
'co2_cont_temp': 0.01,
'nadir_pt': 0.01,
'srf_pressure': 0.001,
'srf_temp_est': 0.01},
'BOL': {'brightness_temp_bol': 0.01,
'tbol': 0.000152587890625,
'vbol': 0.000152587890625},
'CMP': {},
'GEO': {'angular_semidiameter': 0.01,
'emission': 0.01,
'height': 0.01,
'heliocentric_lon': 0.01,
'incidence': 0.01,
'latitude': 0.01,
'local_time': 0.001,
'longitude': 0.01,
'phase': 0.01,
'planetary_phase': 0.01,
'solar_distance': 10000,
'sub_sc_lat': 0.01,
'sub_sc_lon': 0.01,
'sub_solar_lat': 0.01,
'sub_solar_lon': 0.01},
'IFG': {},
'OBS': {'pnt_angle': 0.046875, 'temps': 0.01},
'PCT': {},
'POS': {},
'RAD': {'target_temp': 0.01, 'tdet': 0.01},
'TLM': {'V1': 3.90625,
'V10': -0.15625,
'V11': 0.0976055,
'V12': -0.0985813,
'V13': 0.976562,
'V14': 0.0648437,
'V15': 0.045727,
'V16': 0.0480992,
'V17': 0.0478277,
'V18': 0.0488039,
'V19': 0.141966,
'V2': 1.95312,
'V20': -0.149688,
'V3': 0.278906,
'V4': 0.278906,
'V5': 4.45312,
'V6': 0.652344,
'V7': 0.119457,
'V8': -0.103067,
'V9': 0.15576,
'aux_temps': 0.01,
'ifgm_max': 0.000152587890625,
'ifgm_min': 0.000152587890625}}
plio/io/io_tes.py 0 → 100644
+ 284
− 0
View file @ 0e520231
import numpy as np
import pandas as pd
import pvl
import pvl
import sys
import functools
import json
from shapely.geometry import Polygon
import sqlalchemy
from glob import glob
from os.path import dirname
from shutil import copyfile
from plio.io.io_json import read_json
from plio.io import tes_dtype_map
from plio.io import tes_columns
from plio.io import tes_scaling_factors
class Tes(object):
"""
Attributes
----------
spectra : panel
A pandas panel containing n individual spectra.
ancillary_data : dataframe
A pandas DataFrame of the parsed ancillary data (PVL label)
label : object
The raw PVL label object
"""
def __init__(self, input_data, var_file = None):
"""
Read the .spc file, parse the label, and extract the spectra
Parameters
----------
input_data : string
The PATH to the input .tab file
"""
def expand_column(df, expand_column, columns):
array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
del new_df[expand_column]
return new_df
def bolquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
arr = np.array(lis, dtype=types)
return arr
def obsquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def obsclass2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
bitarr2int(bitarr[16:]))]
types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
arr = np.array(lis, dtype=types)
return arr
def radquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
bitarr2int(bitarr[8:9]))]
types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
('DETECTOR_MASK_PROBLEM', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def atmquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def expand_column(df, expand_column, columns):
array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
del new_df[expand_column]
return new_df
def bolquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
arr = np.array(lis, dtype=types)
return arr
def obsquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def obsclass2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
bitarr2int(bitarr[16:]))]
types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
arr = np.array(lis, dtype=types)
return arr
def radquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
bitarr2int(bitarr[8:9]))]
types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
('DETECTOR_MASK_PROBLEM', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def atmquality2arr(arr):
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def tes2numpy(msb_type, num_bytes, nelems=1):
"""
Converts a MSB data type to a numpy datatype
"""
valid_bytes = {
'MSB_UNSIGNED_INTEGER': [1,2,4,8,16,32,64],
'MSB_INTEGER': [1,2,4,8,16,32,64],
'IEEE_REAL': [1,2,4,8,16,32,64],
'CHARACTER': range(1,128),
'MSB_BIT_STRING': range(1,128)
}
msb_bit_string_type = [('byte{}'.format(i), '>u1') for i in range(num_bytes)]
dtype_map = {
'MSB_UNSIGNED_INTEGER': '>u{}'.format(num_bytes),
'MSB_INTEGER': '>i{}'.format(num_bytes),
'IEEE_REAL': '>f{}'.format(num_bytes),
'CHARACTER': 'a{}'.format(num_bytes),
'MSB_BIT_STRING': msb_bit_string_type
}
if num_bytes not in valid_bytes[msb_type] and not nelems:
raise Exception('invalid byte ({}) count for type ({})'.format(num_bytes, msb_type))
if nelems > 1:
# Must be an array
return [('elem{}'.format(i), dtype_map[msb_type]) for i in range(nelems)]
return dtype_map[msb_type]
def bitarr2int(arr):
arr = "".join(str(i) for i in arr)
return np.uint8(int(arr,2))
def bit2bool(bit):
return np.bool_(bit)
def expand_bitstrings(df, dataset):
if dataset == 'BOL':
quality_columns = ['ti_bol_rating', 'bol_ref_lamp']
df['quality'] = df['quality'].apply(bolquality2arr)
return expand_column(df, 'quality', quality_columns)
elif dataset == 'OBS':
quality_columns = ['hga_motion', 'pnl_motion', 'algor_patch', 'imc_patch',
'momentum', 'equal_tab']
class_columns = ['phase', 'type', 'sequence',
'lamp_status', 'timing', 'spare', 'class_value']
df['quality'] = df['quality'].apply(obsquality2arr)
df['class'] = df['class'].apply(obsclass2arr)
new_df = expand_column(df, 'quality', quality_columns)
new_df = expand_column(new_df, 'class', class_columns)
return new_df
elif dataset == 'RAD':
quality_columns = ['phase_inversion', 'algor_risk', 'calib_fail', 'calib_quality',
'spect_noise', 'ti_spc_rating', 'det_mask_problem']
df['quality'] = df['quality'].apply(radquality2arr)
return expand_column(df, 'quality', quality_columns)
elif dataset == 'ATM':
quality_columns = ['atm_pt_rating', 'atm_opacity_rating']
df['quality'] = df['quality'].apply(atmquality2arr)
return expand_column(df, 'quality', quality_columns)
else:
return df
self.label = pvl.load(input_data)
nrecords = self.label['TABLE']['ROWS']
nbytes_per_rec = self.label['RECORD_BYTES']
data_start = self.label['LABEL_RECORDS'] * self.label['RECORD_BYTES']
dataset = self.label['TABLE']['^STRUCTURE'].split('.')[0]
numpy_dtypes = tes_dtype_map
columns = tes_columns
scaling_factors = tes_scaling_factors
with open(input_data, 'rb') as file:
file.seek(data_start)
buffer = file.read(nrecords*nbytes_per_rec)
array = np.frombuffer(buffer, dtype=numpy_dtypes[dataset.upper()]).byteswap().newbyteorder()
df = pd.DataFrame(data=array, columns=columns[dataset.upper()])
# Read Radiance array if applicable
if dataset.upper() == 'RAD':
with open('{}.var'.format(path.splitext(f)[0]) , 'rb') as file:
buffer = file.read()
def process_rad(index):
if index is -1:
return None
length = np.frombuffer(buffer[index:index+2], dtype='>u2')[0]
exp = np.frombuffer(buffer[index+2:index+4], dtype='>i2')[0]
radarr = np.frombuffer(buffer[index+4:index+4+length-2], dtype='>i2') * (2**(exp-15))
if np.frombuffer(buffer[index+4+length-2:index+4+length], dtype='>u2')[0] != length:
warnings.warn("Last element did not match the length for file index {} in file {}".format(index, f))
return radarr
df["raw_rad"] = df["raw_rad"].apply(process_rad)
df["cal_rad"] = df["cal_rad"].apply(process_rad)
# Apply scaling factors
for column in scaling_factors[dataset]:
def scale(x):
return np.multiply(x, scaling_factors[dataset][column])
df[column] = df[column].apply(scale)
df = expand_bitstrings(df, dataset.upper())
self.data = df
plio/io/tests/test_io_spectral_profiler.py
+ 2
− 2
View file @ 0e520231
......@@ -11,10 +11,10 @@ from plio.io import io_spectral_profiler
from plio.io.io_gdal import GeoDataset
class Test_Spectral_Profiler_IO(unittest.TestCase):
def setUp(self):
self.examplefile = get_path('SP_2C_02_02358_S138_E3586.spc')
def test_openspc(self):
ds = io_spectral_profiler.Spectral_Profiler(self.examplefile)
self.assertEqual(ds.nspectra, 38)
......
plio/io/tests/test_io_tes.py 0 → 100644
+ 25
− 0
View file @ 0e520231
import os
import sys
import unittest
import pandas as pd
sys.path.insert(0, os.path.abspath('..'))
from plio.examples import get_path
from plio.io import io_tes
from plio.io.io_gdal import GeoDataset
class Test_Tes_IO(unittest.TestCase):
def setUp(self):
self.examplefile = get_path('pos10001.tab')
def test_openspc(self):
ds = io_tes.Tes(self.examplefile)
self.assertEqual(ds.data.size, 119106)
self.assertIsInstance(ds.data, pd.DataFrame)
self.assertEqual(ds.data.columns.tolist(), ['sclk_time', 'et', 'pos', 'sun', 'quat', 'id'])
if __name__ == '__main__':
unittest.main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment