Moved io_jsc from PySAT into plio

plio/io/io_jsc.py

+import os
+
+import numpy as np
+import pandas as pd
+from pandas.core.common import array_equivalent
+
+from plio.utils.utils import file_search
+
+
+# This function reads the lookup tables used to expand metadata from the file names
+# This is separated from parsing the filenames so that for large lists of files the
+# lookup tables don't need to be read over and over
+#
+# Info in the tables is stored in a dict of dataframes so that only one variable
+# (the dict) needs to be passed between functions
+def read_refdata(LUT_files):
+    ID_info = pd.read_csv(LUT_files['ID'], index_col=0)
+    spectrometer_info = pd.read_csv(LUT_files['spect'], index_col=0)
+    # spectrometer_info.reset_index(inplace=True)
+    laser_info = pd.read_csv(LUT_files['laser'], index_col=0)
+    # laser_info.reset_index(inplace=True)
+    exp_info = pd.read_csv(LUT_files['exp'], index_col=0)
+    # exp_info.reset_index(inplace=True)
+    sample_info = pd.read_csv(LUT_files['sample'], index_col=0)
+    # sample_info.reset_index(inplace=True)
+    refdata = {'spect': spectrometer_info, 'laser': laser_info, 'exp': exp_info, 'sample': sample_info, 'ID': ID_info}
+    return refdata
+
+
+# This function parses the file names to record metadata related to the observation
+def jsc_filename_parse(filename, refdata):
+    filename = os.path.basename(filename)  # strip the path off of the file name
+    filename = filename.split('_')  # split the file name on underscores
+    libs_ID = filename[0]
+    laserID = filename[4][0]
+    expID = filename[5]
+    spectID = filename[6]
+
+    try:
+        sampleID = refdata['ID'].loc[libs_ID].values[0]
+        file_info = pd.DataFrame(refdata['sample'].loc[sampleID])
+        if file_info.columns.shape[0] < file_info.index.shape[0]:
+            file_info = file_info.T
+        if file_info.index.shape[0] > 1:
+            print('More than one matching row for ' + sampleID + '!')
+            tempID = 'Unknown'
+            file_info = pd.DataFrame(refdata['sample'].loc[tempID])
+            if file_info.columns.shape[0] < file_info.index.shape[0]:
+                file_info = file_info.T
+
+
+    except:
+        sampleID = 'Unknown'
+        file_info = pd.DataFrame(refdata['sample'].loc[sampleID])
+        if file_info.columns.shape[0] < file_info.index.shape[0]:
+            file_info = file_info.T
+
+    file_info['Sample ID'] = sampleID
+    file_info['LIBS ID'] = libs_ID
+    file_info.reset_index(level=0, inplace=True, drop=True)
+    file_info['loc'] = int(filename[1])
+    file_info['lab'] = filename[2]
+    file_info['gas'] = filename[3][0]
+    file_info['pressure'] = float(filename[3][1:])
+
+    if laserID in refdata['laser'].index:
+        laser_info = pd.DataFrame(refdata['laser'].loc[laserID]).T
+        laser_info.index.name = 'Laser Identifier'
+        laser_info.reset_index(level=0, inplace=True)
+        file_info = pd.concat([file_info, laser_info], axis=1)
+
+    file_info['laser_power'] = float(filename[4][1:])
+    if expID in refdata['exp'].index:
+        exp_info = pd.DataFrame(refdata['exp'].loc[expID]).T
+        exp_info.index.name = 'Exp Identifier'
+        exp_info.reset_index(level=0, inplace=True)
+        file_info = pd.concat([file_info, exp_info], axis=1)
+
+    file_info['spectrometer'] = spectID
+    if spectID in refdata['spect'].index:
+        temp = refdata['spect'].loc[spectID]
+        temp = [temp[2], temp[4:]]
+        spect_info = pd.DataFrame(refdata['spect'].loc[spectID]).T
+        spect_info.index.name = 'Spectrometer Identifier'
+        spect_info.reset_index(level=0, inplace=True)
+        file_info = pd.concat([file_info, spect_info], axis=1)
+
+    return file_info
+
+
+def JSC(input_files, refdata):
+    try:
+        # read the first file
+        data = pd.read_csv(input_files[0], skiprows=14, sep='\t', engine='c')
+        data = data.rename(columns={data.columns[0]: 'time1', data.columns[1]: 'time2'})
+        metadata = pd.concat([jsc_filename_parse(input_files[0], refdata)] * len(data.index))
+        metadata.drop('spectrometer', axis=1, inplace=True)
+
+        # read the next files and merge them with the first
+        for file in input_files[1:]:
+            datatemp = pd.read_csv(file, skiprows=14, sep='\t', engine='c')
+            datatemp = datatemp.rename(columns={datatemp.columns[0]: 'time1', datatemp.columns[1]: 'time2'})
+            data = data.merge(datatemp)
+
+        time = data[['time1', 'time2']]  # split the two time columns from the data frame
+        data.drop(['time1', 'time2'], axis=1, inplace=True)  # trim the data frame so it is just the spectra
+
+        # make a multiindex for each wavlength column so they can be easily isolated from metadata later
+        data.columns = [['wvl'] * len(data.columns), np.array(data.columns.values, dtype='float').round(4)]
+
+        metadata.index = data.index
+        metadata = pd.concat([metadata, time], axis=1)
+        compcols = ['SiO2', 'TiO2', 'Al2O3', 'Cr2O3', 'Fe2O3T', 'MnO', 'MgO', 'CaO', 'Na2O', 'K2O', 'P2O5',
+                    'SO3 LOI Residue', 'Total', 'Total Includes', '%LOI', 'FeO',
+                    'Fe2O3', 'SO3 Actual', 'Fe(3+)/Fe(Total)', 'Rb (ug/g)', 'Sr (ug/g)', 'Y (ug/g)', 'Zr (ug/g)',
+                    'V (ug/g)', 'Ni (ug/g)', 'Cr (ug/g)',
+                    'Nb (ug/g)', 'Ga (ug/g)', 'Cu (ug/g)', 'Zn (ug/g)', 'Co (ug/g)', 'Ba (ug/g)', 'La (ug/g)',
+                    'Ce (ug/g)', 'U (ug/g)', 'Th (ug/g)', 'Sc (ug/g)',
+                    'Pb (ug/g)', 'Ge (ug/g)', 'As (ug/g)', 'Cl (ug/g)']
+        compdata = metadata[compcols]
+        metadata.drop(compcols, axis=1, inplace=True)
+        metadata.columns = [['meta'] * len(metadata.columns), metadata.columns.values]
+        compdata.columns = [['comp'] * len(compdata.columns), compdata.columns.values]
+        data = pd.concat([data, metadata, compdata], axis=1)
+
+        data[('meta', 'Scan #')] = data.index
+        data.set_index(('meta', 'time2'), drop=False, inplace=True)
+
+        return data
+    except:
+        print('Problem reading:' + input_file)
+        print('Moving to Problem_Files')
+        os.rename(input_file,
+                  r"Problem_Files\\" + os.path.basename(
+                      input_file))
+        return None
+
+
+def jsc_batch(directory, LUT_files, searchstring='*.txt', to_csv=None):
+    # Read in the lookup tables to expand filename metadata
+    refdata = read_refdata(LUT_files)
+    # get the list of files that match the search string in the given directory
+    filelist = file_search(directory, searchstring)
+    spectIDs = []  # create an empty list to hold the spectrometer IDs
+    libsIDs = []
+    timestamps = []
+    locs = []
+    for file in filelist:
+        filesplit = os.path.basename(file).split('_')
+        spectIDs.append(filesplit[6])  # get the spectrometer IDs for each file in the list
+        libsIDs.append(filesplit[0])
+        timestamps.append(filesplit[-1].split('.')[0])
+        locs.append(filesplit[1])
+    spectIDs_unique = np.unique(spectIDs)  # get the unique spectrometer IDs
+    libsIDs_unique = np.unique(libsIDs)
+    dfs = []  # create an empty list to hold the data frames for each spectrometer
+
+    # loop through each LIBS ID
+    alldata = []
+    for ID in libsIDs_unique:
+        print('Working on : ' + str(ID))
+        sublist = filelist[np.in1d(libsIDs, ID)]
+        locs = []
+        for file in sublist:
+            locs.append(os.path.basename(file).split('_')[1])
+        locs_unique = np.unique(locs)
+        # loop through each location for that libs ID
+        for loc in locs_unique:
+            print(loc)
+            sub_sublist = sublist[np.in1d(locs, loc)]  # get the files for that LIBSID and location
+            data = JSC(sub_sublist, refdata)
+            alldata.append(data)
+            pass
+
+    combined = pd.concat(alldata)
+    if to_csv is not None:
+        print('Writing combined data to: ' + to_csv)
+        combined.to_csv(to_csv)
+    return combined
+
+
+# got this function from stack overflow: http://stackoverflow.com/questions/14984119/python-pandas-remove-duplicate-columns
+# it's slow but doesn't crash python like combined.T.drop_duplicates().T does in some cases with very large sets of data
+def duplicate_columns(frame):
+    groups = frame.columns.to_series().groupby(frame.dtypes).groups
+    dups = []
+
+    for t, v in groups.items():
+
+        cs = frame[v].columns
+        vs = frame[v]
+        lcs = len(cs)
+
+        for i in range(lcs):
+            ia = vs.iloc[:, i].values
+            for j in range(i + 1, lcs):
+                ja = vs.iloc[:, j].values
+                if array_equivalent(ia, ja):
+                    dups.append(cs[i])
+                    break
+
+    return dups