{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tthatcher/anaconda3/envs/autocnet/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n"
]
}
],
"source": [
"\n",
"import os\n",
"import sys\n",
"sys.path.insert(0, os.path.abspath('/home/tthatcher/Desktop/Projects/Plio/plio'))\n",
"\n",
"import warnings\n",
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"from functools import singledispatch\n",
"from plio.examples import get_path\n",
"from plio.io.io_bae import read_gpf"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [],
"source": [
"def read_atf(atf_file):\n",
" \n",
" with open(atf_file) as f:\n",
" \"\"\"\n",
" Read an .atf file and return a dict with .sup, .ipf, .prj, .gpf and\n",
" the path to the directory containing the files.\n",
"\n",
" Parameters\n",
" ----------\n",
" input_data : file\n",
" .atf file\n",
"\n",
" Returns\n",
" -------\n",
" files : Python Dictionary\n",
" containing lists of the files mentioned above\n",
" \"\"\"\n",
" \n",
" from collections import defaultdict\n",
"\n",
" files = defaultdict(list)\n",
"\n",
" for line in f:\n",
" filename = os.path.splitext(line)[0]\n",
" ext = os.path.splitext(line)[1]\n",
" \n",
" # Grabs all the IPF and appends them to files['GP_FILE]\n",
" if(ext == '.ipf\\n'):\n",
" files['IMAGE_IPF'].append(filename.split(' ')[1] + ext.strip())\n",
" \n",
" # Grabs all the GPF and appends them to files['GP_FILE]\n",
" if ext == '.gpf\\n':\n",
" files['GP_FILE'].append(filename.split(' ')[1] + ext.strip())\n",
" \n",
" # Grabs all the PRJ and appends them to files['GP_FILE]\n",
" if ext == '.prj\\n':\n",
" files['PROJECT'].append(filename.split(' ')[1] + ext.strip())\n",
" \n",
" # Grabs all the SUP and appends them to files['GP_FILE]\n",
" if ext == '.sup\\n':\n",
" files['IMAGE_SUP'].append(filename.split(' ')[1] + ext.strip())\n",
" \n",
" # Gets the filepath of the ATF file and stores it in files['basepath]\n",
" files['basepath'] = os.path.dirname(os.path.abspath(atf_file))\n",
" \n",
" return files\n",
"\n",
"@singledispatch\n",
"def read_ipf(arg):\n",
" return str(arg)\n",
"\n",
"@read_ipf.register(str)\n",
"def read_ipf_str(input_data):\n",
" \"\"\"\n",
" Read a socet ipf file into a pandas data frame\n",
"\n",
" Parameters\n",
" ----------\n",
" input_data : str\n",
" path to the an input data file\n",
"\n",
" Returns\n",
" -------\n",
" df : pd.DataFrame\n",
" containing the ipf data with appropriate column names and indices\n",
" \"\"\"\n",
"\n",
" # Check that the number of rows is matching the expected number\n",
" with open(input_data, 'r') as f:\n",
" for i, l in enumerate(f):\n",
" if i == 1:\n",
" cnt = int(l)\n",
" elif i == 2:\n",
" col = l\n",
" break\n",
" \n",
" columns = np.genfromtxt(input_data, skip_header=2, dtype='unicode',\n",
" max_rows = 1, delimiter = ',')\n",
"\n",
" # TODO: Add unicode conversion\n",
" d = [line.split() for line in open(input_data, 'r')]\n",
" d = np.hstack(np.array(d[3:]))\n",
" \n",
" d = d.reshape(-1, 12)\n",
" \n",
" df = pd.DataFrame(d, columns=columns)\n",
" df['ipf_file'] = pd.Series(np.full((len(df['pt_id'])), input_data), index = df.index)\n",
" print(df['ipf_file'])\n",
"\n",
" assert int(cnt) == len(df), 'Dataframe length {} does not match point length {}.'.format(int(cnt), len(df))\n",
" \n",
" # Soft conversion of numeric types to numerics, allows str in first col for point_id\n",
" df = df.apply(pd.to_numeric, errors='ignore')\n",
"\n",
" return df\n",
"\n",
"@read_ipf.register(list)\n",
"def read_ipf_list(input_data_list):\n",
" \"\"\"\n",
" Read a socet ipf file into a pandas data frame\n",
"\n",
" Parameters\n",
" ----------\n",
" input_data_list : list\n",
" list of paths to the a set of input data files\n",
"\n",
" Returns\n",
" -------\n",
" df : pd.DataFrame\n",
" containing the ipf data with appropriate column names and indices\n",
" \"\"\"\n",
" frames = []\n",
"\n",
" for input_file in input_data_list:\n",
" frames.append(read_ipf(input_file))\n",
"\n",
" df = pd.concat(frames)\n",
"\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 107,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/home/tthatcher/Desktop/Projects/Plio/plio/plio/examples/SocetSet\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"81 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"82 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"83 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"84 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"85 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"86 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"87 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"88 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"89 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"90 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"91 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"92 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"93 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"94 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"95 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"96 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"97 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"98 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"99 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"100 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"101 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"102 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"103 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"104 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"105 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"106 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"107 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"108 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"109 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"110 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 111, dtype: object\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"91 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"92 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"93 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"94 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"95 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"96 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"97 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"98 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"99 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"100 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"101 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"102 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"103 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"104 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"105 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"106 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"107 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"108 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"109 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"110 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"111 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"112 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"113 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"114 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"115 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"116 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"117 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"118 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"119 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"120 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 121, dtype: object\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"144 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"145 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"146 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"147 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"148 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"149 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"150 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"151 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"152 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"153 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"154 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"155 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"156 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"157 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"158 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"159 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"160 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"161 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"162 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"163 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"164 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"165 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"166 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"167 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"168 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"169 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"170 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"171 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"172 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"173 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 174, dtype: object\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"144 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"145 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"146 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"147 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"148 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"149 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"150 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"151 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"152 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"153 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"154 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"155 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"156 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"157 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"158 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"159 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"160 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"161 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"162 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"163 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"164 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"165 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"166 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"167 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"168 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"169 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"170 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"171 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"172 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"173 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 174, dtype: object\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"134 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"135 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"136 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"137 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"138 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"139 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"140 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"141 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"142 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"143 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"144 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"145 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"146 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"147 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"148 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"149 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"150 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"151 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"152 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"153 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"154 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"155 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"156 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"157 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"158 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"159 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"160 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"161 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"162 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"163 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 164, dtype: object\n",
"0 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"1 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"2 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"3 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"4 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"5 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"6 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"7 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"8 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"9 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"10 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"11 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"12 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"13 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"14 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"15 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"16 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"17 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"18 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"19 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"20 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"21 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"22 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"23 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"24 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"25 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"26 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"27 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"28 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"29 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
" ... \n",
"172 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"173 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"174 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"175 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"176 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"177 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"178 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"179 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"180 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"181 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"182 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"183 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"184 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"185 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"186 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"187 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"188 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"189 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"190 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"191 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"192 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"193 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"194 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"195 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"196 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"197 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"198 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"199 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"200 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"201 /home/tthatcher/Desktop/Projects/Plio/plio/pli...\n",
"Name: ipf_file, Length: 202, dtype: object\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tthatcher/anaconda3/envs/autocnet/lib/python3.6/site-packages/ipykernel_launcher.py:13: UserWarning: The following points found in ipf files missing from gpf file: \n",
"\n",
"['P03_002226_1895_XI_09N203W_15', 'P03_002226_1895_XI_09N203W_16', 'P03_002226_1895_XI_09N203W_17', 'P03_002226_1895_XI_09N203W_18', 'P03_002226_1895_XI_09N203W_19', 'P03_002226_1895_XI_09N203W_20', 'P03_002226_1895_XI_09N203W_21', 'P03_002226_1895_XI_09N203W_22', 'P03_002226_1895_XI_09N203W_24', 'P03_002226_1895_XI_09N203W_26', 'P03_002226_1895_XI_09N203W_30', 'P03_002226_1895_XI_09N203W_31', 'P03_002226_1895_XI_09N203W_32', 'P03_002226_1895_XI_09N203W_34', 'P03_002226_1895_XI_09N203W_36', 'P03_002226_1895_XI_09N203W_37', 'P03_002226_1895_XI_09N203W_44', 'P03_002226_1895_XI_09N203W_48', 'P03_002226_1895_XI_09N203W_49', 'P03_002226_1895_XI_09N203W_56', 'P03_002226_1895_XI_09N203W_57', 'P03_002226_1895_XI_09N203W_61', 'P03_002226_1895_XI_09N203W_62', 'P03_002226_1895_XI_09N203W_63', 'P03_002226_1895_XI_09N203W_65', 'P19_008344_1894_XN_09N203W_4', 'P20_008845_1894_XN_09N203W_15']. \n",
"\n",
"Continuing, but these points will be missing from the control network\n",
" del sys.path[0]\n"
]
},
{
"data": {
"text/plain": [
"['val',\n",
" 'fid_val',\n",
" 'no_obs',\n",
" 'l.',\n",
" 's.',\n",
" 'sig_l',\n",
" 'sig_s',\n",
" 'res_l',\n",
" 'res_s',\n",
" 'fid_x',\n",
" 'fid_y',\n",
" 'ipf_file',\n",
" 'stat',\n",
" 'known',\n",
" 'lat_Y_North',\n",
" 'long_X_East',\n",
" 'ht',\n",
" 'sig0',\n",
" 'sig1',\n",
" 'sig2',\n",
" 'res0',\n",
" 'res1',\n",
" 'res2']"
]
},
"execution_count": 107,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"atf_dict = read_atf(get_path('CTX_Athabasca_Middle_step0.atf'))\n",
"\n",
"print(atf_dict['basepath'])\n",
"gpf_file = os.path.join(atf_dict['basepath'], atf_dict['GP_FILE'][0])\n",
"ipf_list = [os.path.join(atf_dict['basepath'], i) for i in atf_dict['IMAGE_IPF']]\n",
"\n",
"gpf_df = read_gpf(gpf_file).set_index('point_id')\n",
"ipf_df = read_ipf(ipf_list).set_index('pt_id')\n",
"\n",
"point_diff = ipf_df.index.difference(gpf_df.index)\n",
"\n",
"if len(point_diff) != 0:\n",
" warnings.warn(\"The following points found in ipf files missing from gpf file: \\n\\n{}. \\\n",
" \\n\\nContinuing, but these points will be missing from the control network\".format(list(point_diff)))\n",
"\n",
"new_df = ipf_df.merge(gpf_df, left_index=True, right_index=True)\n",
"list(new_df)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'lines' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-fc95c78d29ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefaultdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplitext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mext\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'lines' is not defined"
]
}
],
"source": [
"from collections import defaultdict\n",
"\n",
"files = defaultdict(list)\n",
"\n",
"for line in lines:\n",
" ext = os.path.splitext(line)[-1]\n",
" files[ext].append(line)\n",
"\n",
"files['basepath'] = os.path.dirname(os.path.abspath(atf_file))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}