/**
 * @file
 * $Revision: 1.5 $
 * $Date: 2010/04/09 21:11:43 $
 *
 *   Unless noted otherwise, the portions of Isis written by the USGS are
 *   public domain. See individual third-party library and package descriptions
 *   for intellectual property information, user agreements, and related
 *   information.
 *
 *   Although Isis has been used by the USGS, no warranty, expressed or
 *   implied, is made by the USGS as to the accuracy and functioning of such
 *   software and related material nor shall the fact of distribution
 *   constitute any such warranty, and no responsibility is assumed by the
 *   USGS in connection therewith.
 *
 *   For additional information, launch
 *   $ISISROOT/doc//documents/Disclaimers/Disclaimers.html
 *   in a browser or see the Privacy &amp; Disclaimers page on the Isis website,
 *   http://isis.astrogeology.usgs.gov, and the USGS privacy and disclaimers on
 *   http://www.usgs.gov/privacy.html.
 */
#include <string>
#include <vector>
#include <numeric>
#include <iostream>
#include <sstream>
#include "IString.h"
#include "CSVReader.h"
#include "CollectorMap.h"
#include "IException.h"

using namespace std;

namespace Isis {

  /**
   * @brief Default constructor for CSV reader
   *
   * The default constructor sets up to read a source that has not header and
   * skips no lines.  It also sets the delimiter to the comma, as implied by its
   * name (CSV = comma separated value), and treats multiple successive occurances
   * of the delimiting character as individual tokens (keeping empty parts).
   *
   * This method can be used when deferring the reading of the input source.
   * Other methods available in this class can be used to adjust the behavior of
   * the parsing before [i]and[/i] after reading of the source as parsing is
   * performed on demand.  This means a single input source can be parsed
   * repeatedly after adjusting parameters.
   */
  CSVReader::CSVReader() : _header(false), _skip(0),
    _delimiter(','), _keepParts(true), _lines(),
    _ignoreComments(true) { }

  /**
   * @brief Parameterized constructor for parsing an input file source
   *
   * This constructor can be used when the input source is an identified file.
   * Parameters are available for specifying the parsing behavior, but are not
   * necessarily required here as defaults are provided.  Other methods in this
   * class can set parsing conditions after the input file has been read in.
   *
   * If the file cannot be opened or an error is encountered during the reading of
   * the file, an Isis exception is thrown.
   *
   * All lines are read in from the file and stored for subsequent parsing.
   * Therefore, parsing can be performed at any time upon returning from this
   * constructor.
   *
   * @param csvfile  Name of file to open and read
   * @param header   Indicates if a header exists (true) in the file or not
   *                 (false)
   * @param skip  Number of lines to skip to header, if it exists, or to the first
   *              data line
   * @param delimiter  Indicates the character to be used to delimit each token in
   *                   the string/line
   * @param keepEmptyParts  Indicates successive delimiters are to be treated as
   *                        empty tokens (true) or collapsed into one token
   *                        (false)
   */
  CSVReader::CSVReader(const QString &csvfile, bool header, int skip,
                       const char &delimiter, const bool keepEmptyParts,
                       const bool ignoreComments) :
    _header(header), _skip(skip), _delimiter(delimiter),
    _keepParts(keepEmptyParts), _lines(),
    _ignoreComments(ignoreComments) {

    m_csvFile = csvfile;
    read(csvfile);
  }


  /**
   * @brief Determine the number of columns in the input source
   *
   * This method is applies the parsing conditions to all data lines to determine
   * the number of columns.  Note that it is assumed that all lines contain the
   * same number of columns.
   *
   * If the number of columns vary in any of the lines, the least number of
   * columns found in all lines is returned due to the nature of how the columns
   * are determined.  @see isTableValid().
   *
   * Note that this can be an expensive operation if the input source is large as
   * all lines are parsed.  This does not include the header.  @see columns(const
   * CSVReader::CSVTable &table) for an alternative and more efficient method.
   * That method takes a previously parsed table of all lines as an argument,
   * which is precisely how this method determines the columns.
   *
   * @return int Number of columns in table, smallest column count if some lines
   *         are different
   * @see getColumnSummary()
   */
  int CSVReader::columns()  const {
    return ((rows() > 0) ? columns(getTable()) : 0);
  }

  /**
   * @brief Determine the number of columns in a parser CSV Table
   *
   * This method computes the number of columns from a CSVTable.  This table is a
   * result of the getTable method.
   *
   * It is assumed each row in the table has the same number of columns after
   * parsing.  If one or more of the rows contain differing columns, only the
   * smallest number of columns are reported.
   *
   * @param table The table from which the CVSTable rows are obtained
   *
   * @return int  Number of columns in table, smallest column count if some lines
   *         are different
   * @see getColumnSummary()
   */
  int CSVReader::columns(const CSVReader::CSVTable &table) const {
    CSVColumnSummary summary = getColumnSummary(table);
    return ((summary.size() > 0) ? summary.key(0) : 0);
  }


  /**
   * @brief Reads the entire contents of a file for subsequent parsing
   *
   * This method opens the specified file and reads every line storing them in
   * this object.  It is assumed this file is a text file.  Other methods in this
   * class can be utilized to set parsing conditions before [i]or[/i] after the
   * file has been read.
   *
   * Note that parsing the file is deferred until explicity invoked through other
   * methods in this class.  Users of this class can extract individual rows,
   * columns or the complete table.
   *
   * This object is reentrant.  Additional files can be read in.  Any existing
   * data from previous input sources is discarded upon subsequent reads.
   *
   * @param csvfile  Name of file to read
   */
  void CSVReader::read(const QString &csvfile) {
    ifstream ifile(csvfile.toLatin1().data(), ios::in);
    if(!ifile) {
      QString mess = "Unable to open file " + csvfile;
      throw IException(IException::User, mess, _FILEINFO_);
    }
    
    m_csvFile = csvfile;
    _lines.clear();
    load(ifile);
    ifile.close();
  }

  /**
   * @brief Retrieve the header from the input source if it exists
   *
   * This method will return the header if it exists after appling the parsing
   * rules.
   *
   * The existance of the header is determined entirely by the user of this class.
   * If the header does not exist, a zero-length array is returned.
   *
   * Note that this routine does not trim leading or trailing whitespace from each
   * header.  This must be handled by the caller.
   *
   * @return CSVReader::CSVAxis  Array containing the elements of the header
   * @see haveHeader()
   * @see setHeader()
   */
  CSVReader::CSVAxis CSVReader::getHeader() const{
    //  Return an empty header if we don't have one
    if((!_header) || (_skip >= rows())) {
      return (CSVAxis(0));
    }
    return (Parser(_lines[_skip], _delimiter, _keepParts).result());
  }
  
  /**
   * @brief This method will return the index at which name occurs.
   * 
   * @param name The column name we are trying to find.
   * 
   * @throws IException::Programmer This file does not have a header set.
   *
   * @return @b int The index at which the column name occurs, -1 is returned if it does not exist
   */
  int CSVReader::getHeaderColumn(QString name) {
    if (!haveHeader()) {
      QString mess = "This file does not have a header set.";
      throw IException(IException::Programmer, mess, _FILEINFO_);
      return -1;
    }
    QString columnName = QString(name).trimmed();
    
    const CSVReader::CSVAxis header = getHeader();
    
    if(header.dim() <= 0) {
      QString mess = "Could not get a valid header.";
      throw IException(IException::Programmer, mess, _FILEINFO_);
      return -1;
    }
    
    for (int i = 0; i < header.dim(); i++) {
      if (columnName.toLower() == QString(header[i]).toLower().trimmed()) {
        return i;
      }
    }
    QString mess = "Could not find that column header.";
    throw IException(IException::Programmer, mess, _FILEINFO_);
    return -1;    
      
  }
    
  /**
   * @brief Parse and return the requested row by index
   *
   * This method will parse and return the requested row from the input source as
   * an array.  If the requested row is determined to be an invalid index, then a
   * zero-length array is returned.  It is up to the caller to check for validity
   * of the returned row array.
   *
   * @param index  Index of the desired row to return
   *
   * @return CSVReader::CSVAxis  Array of tokens after parsing rules are applied
   */
  CSVReader::CSVAxis CSVReader::getRow(int index) const{
    //  Return an empty header if we don't have one
    if((index < 0) || (index >= rows())) {
      return (CSVAxis(0));
    }
    return (Parser(_lines[index+firstRowIndex()], _delimiter, _keepParts).result());
  }
  
  /**
   * @brief Find and return the row based upon the QPairs in the QList
   *
   * This method will parse and return the requested row from the input source as
   * an array.  If the requested row is determined to be an invalid index, then a
   * zero-length array is returned and an exception is thrown.  
   * 
   * IF there is repeating matching values that match all of the information in imageDataList, this 
   * will return the first row that matches. If you want a unique row then add more search pairs to 
   * imageDataList.
   *
   * @param imageDataList A QList of QPairs that contain first, the name of the column, and second, 
   * the value of the column we are trying to match.
   * 
   * @throws IException::Programmer Cannot search with empty search parameters.
   * @throws IException::Programmer Could not get a valid header index.
   * @throws IException::User Row not found in CSV file.
   *
   * @return @b CSVReader::CSVAxis  Array of tokens after parsing rules are applied, in this
   * case it will be the row that we are trying to find, it will return an empty array if it isn't 
   * found. 
   *         
   */
  CSVReader::CSVAxis CSVReader::getRow(QList< QPair<QString, QString> > imageDataList) {
      
    if (imageDataList.size() <= 0) {
      QString mess = "Cannot search with empty search parameters.";
      throw IException(IException::Programmer, mess, _FILEINFO_);
      return (CSVAxis(0));
    }
      
    QPair<QString, QString> imageNumPair = imageDataList[0];
    
    //Grab the primary pair. The primary pair is the first pair of imageDataList and will be the
    //first column and value compared to the row column/value. To make the search faster, try to 
    //make your first pair the column/value that will eliminate the most rows.
    //
    //For example, when searching for an image based on the image number and the tile number
    //(as in the case of Apollo 15) there are at most 8 rows that match the image number but there
    //are many more rows that match the tile number. So I would use the image number as the primary
    //pair.
    int primaryColumnNum = getHeaderColumn(imageNumPair.first);
    if (primaryColumnNum == -1) {
      QString mess = "Could not get a valid header index.";
      throw IException(IException::Programmer, mess, _FILEINFO_);
      return (CSVAxis(0));
    }
      
    QString primaryValue = imageNumPair.second;
    
    ifstream ifile(m_csvFile.toLatin1().data(), ifstream::in);
    
    //This loops through the file until the row we want is found, or it's the end of file
    //and there are no matching rows
    int j = 0;
    while (!ifile.eof()) {
      const CSVReader::CSVAxis row = getRow(j);
      
      if (row.dim() <= 0) {
        QString mess = "There is no matching data not found in CSV.";
        throw IException(IException::User, mess, _FILEINFO_);
        return (CSVAxis(0));
      }
      
      //Check if the primary pair's value matches the rows value
      if (primaryValue.toLower().trimmed() == QString(row[primaryColumnNum]).toLower().trimmed()) {
        
        //Loops through the rest of imageDataList's pairs and checks the row for column/row 
        //matches
        for (int i = 1; i < imageDataList.size(); i++) {
          QPair<QString, QString> columnPair = imageDataList[i];
          int columnCheckNum = getHeaderColumn(columnPair.first);
          QString columnValue = columnPair.second;
          
          //Check that the columnValue matches the row's column value, if it does and it is the 
          //end of imageDataList then that is the correct row 
          if (columnValue.toLower().trimmed() == QString(row[columnCheckNum]).toLower().trimmed()) {
            if (i == imageDataList.size() - 1) {
              return row;
            }
          }
          //If it doesn't match, break out of this loop because the row doesn't match
          else {
            break;
          }
        }
      }
      j++;
    }
    
    //If we've gotten here then there is no matching row. Tell the user and return an empty token 
    //array.
    QString mess = "Row not found in CSV file.";
    throw IException(IException::User, mess, _FILEINFO_);
    return (CSVAxis(0));
  }
  


  /**
   * @brief Parse and return a column specified by index order
   *
   * This method extracts a column from each row and returns the result.  Note
   * that parsing rules are applied to each row and the column at index is
   * extracted and returned in the array.  The array is always the number of rows
   * from the input source (less skipped lines and header if they exist).
   *
   * It is assumed that every row has the same number of columns (@see
   * isTableValid()) but in the event that the requested column does not exist for
   * any (or all rows for that matter) a default constructed token is returned for
   * that row.  If the requested index is less than 0, an empty column is
   * returned.
   *
   * Columns are 0-based index so the valid number of columns range 0 to
   * (columns() - 1).
   *
   * @param index  Zero-based column index to parse and return
   *
   * @return CSVReader::CSVAxis  Array of token element from each column
   */
  CSVReader::CSVAxis CSVReader::getColumn(int index) const {
    //  Return an empty header if we don't have one
    if(index < 0) {
      return (CSVAxis(0));
    }

    int nrows(rows());
    int nbad(0);
    CSVAxis column(nrows);
    Parser parser;
    for(int i = 0 ; i < nrows ; i++) {
      parser.parse(_lines[i+firstRowIndex()], _delimiter, _keepParts);
      if(parser.size() <= index) {
//      column[i] = Parser::TokenType("");
        nbad++;
      }
      else {
        column[i] = parser(index);
      }
    }

    //  If we had no good columns (index is invalid) return an empty column
    return ((nbad == nrows) ? CSVAxis(0) : column);
  }

  /**
   * @brief Parse and return column specified by header name
   *
   * This method will parse and extract a column that corresponds to named column
   * in the header.  This method return a zero-length array if a header does not
   * exist for this input source or the named column does not exist.
   *
   * The header is parsed using the same rules as each row.  It is the
   * responsibility of the user of this class to specify the existance of a
   * header.  Once the header is parsed, a case-insensitive search of the names is
   * performed until the requested column name is found.  The index of this header
   * name is then used to extract the column from each row.
   *
   * It is assumed the column exists in each row.  If it does not, a default
   * constructed token is returned for non-existant columns in a row.
   *
   * @param hname  Name of the column as it exists in the header
   *
   * @return CSVReader::CSVAxis Column array parsed from each row
   */
  CSVReader::CSVAxis CSVReader::getColumn(const QString &hname) const {
    //  Get the header
    CSVAxis header(getHeader());
    QString head = hname.trimmed();
    for(int i = 0 ; i < header.dim() ; i++) {
      if(head.toLower() == header[i].trimmed().toLower()) {
        return (getColumn(i));
      }
    }

    //  If we reach here, we did not find the column name
    return (CSVAxis(0));
  }


  /**
   * @brief Parse and return all rows and columns in a table array
   *
   * This method returns a 2-D table of all rows and columns after parsing rules
   * are applied.  Each column or token in each row is returned as a
   * CSVParser::TokenType.  Subsequent conversion can be performed if the type
   * sufficiently supports it or the user can provide its own conversion
   * techniques.
   *
   * The validity of the table with regards to column integrity (same number of
   * columns in each row) can be checked with the isTableValid method.  A summary
   * of the number of rows containing differing numbers of columns is provided by
   * the getColumnSummary method.
   *
   * The returned table does not include the header row or any skipped rows.  An
   * empty table, zero-length array is returned if no rows are present.
   *
   * The table itself is a 1-dimenional array that contains a row at each element.
   * This conceptually is a 2-dimensional table.  Each element in the row (first)
   * dimension of the table is a CSVAxis array containing parsed columns or
   * tokens.  Note that the number of columns may vary from row to row.
   *
   * @return CSVReader::CSVTable  2-D table of parsed columns in each row
   */
  CSVReader::CSVTable CSVReader::getTable() const {
    CSVTable table(rows());
    int nrows(rows());
    Parser parser;
    for(int row = 0 ; row < nrows ; row++) {
      parser.parse(_lines[row+firstRowIndex()], _delimiter, _keepParts);
      table[row] = parser.result();
    }
    return (table);
  }

  /**
   * @brief Computes a row summary of the number of distinct columns in table
   *
   * A CSVColumnSummary is a CollectorMap where the key is the number of columns
   * and the value is the number of rows that contain that number of columns.
   * This is useful to determine the consistancy of a parser input source such
   * that every row contains the same number of columns.
   *
   * Once this summary is computed, there should exist one and only ome element in
   * the summary where the key is the column count for each row and the value of
   * that key is the number of rows that contain those columns.
   *
   * This example shows how to determine this information:
   * @code
   *  CSVReader::CSVTable table = csv.getTable();
   *  CSVReader::CSVColumnSummary summary = csv.getColumnSummary(table);
   *  cout << "Number of columns:     " <<  csv.columns(table) << endl;
   *  cout << "Number distinct columns: " << summary.size() << endl;
   *  for (int ncols = 0 ; ncols < summary.size() ; ncols++) {
   *    cout << "--> " << summary.getNth(ncols) << " rows have "
   *         << summary.key(ncols) << " columns." << endl;
   *  }
   * @endcode
   *
   * @param table Input table as returned by the getTable method
   *
   * @return CSVReader::CSVColumnSummary  A CollectorMap that idicates the number
   *         of rows with distinct numbers of columns
   * @see getTable()
   * @see isTableValid()
   */
  CSVReader::CSVColumnSummary CSVReader::getColumnSummary(const CSVTable &table)
  const {
    CSVColumnSummary summary;
    for(int row = 0 ; row < table.dim() ; row++) {
      int n(table[row].dim());
      if(summary.exists(n)) {
        int &count = summary.get(n);
        count++;
      }
      else {
        summary.add(n, 1);
      }
    }

    return (summary);
  }

  /**
   * @brief Indicates if all rows have the same number of columns
   *
   * This method checks the integrity of all rows in the inputs source as to
   * whether they have the same number of columns.
   *
   * @param table  Input table to check for integrity/validty
   *
   * @return bool True if all rows have the same number of columns, false if they
   *         do not
   */
  bool CSVReader::isTableValid(const CSVReader::CSVTable &table) const {
    CSVColumnSummary summary = getColumnSummary(table);
    return (summary.size() <= 1);
  }

  /**
   * @brief Reads all lines from the input stream until an EOF is encoutered
   *
   * This method is the used to read from an input stream all lines of text until
   * an end-of-file (EOF) is encountered.  It is used to perform read operations
   * for all sources of input, files and direct streams as supplied by the users
   * of this class.
   *
   * All lines are assumed to end with a newline sequence pertinent to the systems
   * this software is compiled on.  All lines are stored as they are read in
   * unless they are empty lines.  The default behavior is to treat all lines that
   * begin with a '#' as a comment.  These lines are ignored by default and
   * excluded as they are read.  (Comment and blank line feature was added
   * 2010/04/08.)
   *
   * As lines are read in from the input stream, they are pushed onto the internal
   * stack in the order they are read.  The calling environment is responsible for
   * the state of the stack as to whether it is cleared or appended to an existing
   * state.
   *
   * @param ifile  Input source stream of lines of text
   *
   * @return std::istream& Returns the state of the input stream at the end of
   *         read operations.
   */
  std::istream &CSVReader::load(std::istream &ifile) {

    std::string iline;
    int nlines(0);
    while(getline(ifile, iline)) {
      if(!iline.empty()) {
        if(!(_ignoreComments && (iline[0] == '#'))) {
          _lines.push_back(iline.c_str());
          nlines++;
        }
      }
    }

    if(!ifile.eof()) {
      ostringstream mess;
      mess << "Error reading line " << (nlines + 1) << ends;
      throw IException(IException::User, mess.str(), _FILEINFO_);
    }

    return (ifile);
  }

  /**
   * @brief Input read operator for input stream sources
   *
   * This input operator can be invoked directly from the users environment to
   * read the complete input source.  It can also be used to augment an existing
   * source as this technique does not discard existing data (lines).
   *
   * It is presumed that any additional input sources are consistant to
   * pre-established parsing guidelines otherwise, the integrity of the table is
   * compromized.
   *
   * Here is an example of how to use this method:
   * @code
   * ifstream ifile("myfile.csv");
   * CSVReader csv;
   * ifile >> csv;
   * @endcode
   *
   * @param is  Input stream source
   * @param csv CSVReader object to read input source lines from
   *
   * @return std::istream&  Returns the state of the input stream at EOF or error
   */
  std::istream &operator>>(std::istream &is, CSVReader &csv) {
    return (csv.load(is));
  }

}
