#!/bin/python3

## @package pycompare
#  \brief Script to perform output consistency tests
#
#  Comparing the numeric output can be rendered hard by the amount of information
#  contained in a typical output file and the necessity to determine whether a
#  difference is actually significant or just caused by numeric noise hitting
#  negligible values. The task of `pycompare.py` is to compare two output files, in
#  the assumption that they were written by the FORTRAN and the C++ versions of
#  the code and to flag all the possible inconsistencies according to various
#  severity levels (namely: NOISE, WARNING, and ERROR).
#
#  After execution, the script returns an exit code, which is set to 0, if no
#  error-level inconsistencies were found, or 1 otherwise. This can be used by
#  subsequent system calls to set up a testing suite checking whether the code
#  is able to reproduce legacy results.
#
#  The script execution requires python3.

import re

from math import log10
from sys import argv

## \cond
number_reg = re.compile(r'-?[0-9]\.[0-9]+E[-+][0-9]{2,2}')
## \endcond

## \brief Main execution code
#
# `main()` is the function that handles the creation of the script configuration
# and the execution of the comparison. It returns an integer value corresponding
# to the number of detected error-level inconsistencies.
#
# \returns errors: `int` Number of detected error-level inconsistencies.
def main():
    config = parse_arguments()
    errors, warnings, noisy = (0, 0, 0)
    if config['help_mode'] or len(argv) == 1:
        config['help_mode'] = True
        print_help()
    else:
        compare_log = compare_files(config)
        errors = compare_log['errors']
        warnings = compare_log['warnings']
        noisy = compare_log['noisy']
        print("ERROR COUNT: %d"%errors)
        print("WARNING COUNT: %d"%warnings)
        print("NOISE COUNT: %d"%noisy)
        if (config['log_html']): reformat_log(config, errors, warnings, noisy)
    if (errors > 0):
        print("FAILURE: {0:s} is not consistent with {1:s}".format(
            config['c_file_name'], config['fortran_file_name']
        ))
    else:
        if (not config['help_mode']):
            print("SUCCESS: {0:s} is consistent with {1:s}".format(
                config['c_file_name'], config['fortran_file_name']
            ))
    return errors

## \brief Perform the comparison of two files.
#
#  The comparison is executed as a line-by-line process. In order to process
#  files correctly, it is required that the two input files have exactly the
#  same format (with the exception of some number formatting subtleties that
#  are handled by regular expressions). Therefore, the first comparison step
#  is testing whether the input files have the same number of lines. If this
#  condition is not met, a formatting problem is assumed and every line in
#  the C++ output file is counted as an error. Otherwise, all the numeric
#  values found in the result are compared for consistency within warning
#  tolerance. Warnings and errors are issued if two values do not match by a
#  fractional amount being, respectively, below or above the threshold for
#  warning. A special case is the detection of suspect numeric noise. This
#  arises on very small quantities as a consequence of differences in the
#  level of approximation or in the hardware implementation of the numeric
#  values, which typically has negligible impact on the overall reslults,
#  even though it can have potentially large fractional mismatches, because
#  it is caused by values that are close to 0.
#
#  Numeric noise is filtered by taking advantage from the fact that the
#  output files are formatted in such a way that values with similar physical
#  meaning are written to the same output line. If the comparison results in
#  a large fractional mismatch on a value that is more than 5 orders of
#  magnitude smaller than the highest order of magnitude that was read from
#  the current row, the discrepancy is flagged as a potential noise effect.
#
#  \param config: `dict` A dictionary containing the script configuration.
#
#  \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles
#  together the numbers of detected errors, warnings and noisy values.
def compare_files(config):
    mismatch_count = {
        'errors': 0,
        'warnings': 0,
        'noisy': 0,
    }
    fortran_file = open(config['fortran_file_name'], 'r')
    c_file = open(config['c_file_name'], 'r')
    l_file = None
    f_lines = fortran_file.readlines()
    c_lines = c_file.readlines()
    fortran_file.close()
    c_file.close()
    if (len(f_lines) == len(c_lines)):
        line_count = len(f_lines)
        num_len = 1
        if (line_count > 0):
            num_len = max(4, int(log10(line_count)) + 1)
        if (config['log_html']):
            l_file = open(config['html_output'], 'w')
            l_file.write("<!DOCTYPE html>\n")
            l_file.write("<html xmnls=\"http://www.w3.org/1999/xhtml\">\n")
            l_file.write("  <header>\n")
            l_file.write(
                "    <h1>Comparison between {0:s} and {1:s}</h1>\n".format(
                    config['fortran_file_name'], config['c_file_name']
                )
            )
            l_file.write("  </header>\n")
            l_file.write("  <body>\n")
            l_file.write("    <div>Numeric noise is marked <span style=\"font-weight: bold; color: rgb(0,185,0)\">"
                         + "GREEN</span>, warnings are marked <span style=\"font-weight: bold; color: rgb(0,0,255)\">"
                         + "BLUE</span> and errors are marked <span style=\"font-weight: bold; color: rgb(255,0,0)\">"
                         + "RED</span>.</div>\n")
        for li in range(line_count):
            line_result = compare_lines(f_lines[li], c_lines[li], config, li + 1, num_len, l_file)
            mismatch_count['errors'] += line_result[0]
            mismatch_count['warnings'] += line_result[1]
            mismatch_count['noisy'] += line_result[2]
            if (mismatch_count['errors'] > 0 and not config['check_all']):
                print("INFO: mismatch found at line %d"%(li + 1))
                break
        if l_file is not None:
            l_file.write("  </body>\n")
            l_file.write("</html>\n")
            l_file.close()
    else:
        mismatch_count['errors'] = len(c_lines)
        print("ERROR: {0:s} and {1:s} have different numbers of lines!".format(
            config['fortran_file_name'], config['c_file_name']
        ))
    return mismatch_count

## \brief Perform the comparison of two file lines.
#
#  This function handles the line-by-line comparison of coded result files. Depending
#  on whether a HTML log report was requested, it also undertakes the task of
#  formatting the HTML code to show the comparison results as high-lighted entries,
#  according to the severity degree of the mismatch.
#
#  \param f_line: `string` A line extracted from the FORTRAN output file.
#  \param c_line: `string` A line extracted from the C++ output file.
#  \param config: `dict` A dictionary containing the script configuration.
#  \param line_num: `int` The number of the current line (0-indexed).
#  \param num_len: `int` The number digits to format the line number tag in the HTML log.
#  \param log_file: `file` A file where to write logging information, if required.
#
#  \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles
#  together the numbers of detected errors, warnings and noisy values.
def compare_lines(f_line, c_line, config, line_num=0, num_len=4, log_file=None):
    errors = 0
    warnings = 0
    noisy = 0
    f_line = f_line.replace("D-","E-").replace("D+","E+")
    ref_format = "    <div><span style=\"font-weight: bold; color: rgb(125,125,125)\"><pre><code>{0:%ds}"%num_len
    ref_line = (ref_format + ": {1:s}</code></pre></span></div>\n").format("ORIG", f_line[:-1])
    if (f_line == c_line):
        if log_file is not None:
            if (config['full_log']):
                num_format = "    <div><pre><code>{0:0%dd}"%num_len
                log_line = (num_format + ": {1:s}</code></pre></div>\n").format(line_num, c_line[:-1])
                log_file.write(log_line)
    else:
        iter_f_values = number_reg.finditer(f_line)
        iter_c_values = number_reg.finditer(c_line)
        f_starts, f_ends, f_groups = [], [], []
        c_starts, c_ends, c_groups = [], [], []
        for fi in iter_f_values:
            f_starts.append(fi.start())
            f_ends.append(fi.end())
            f_groups.append(fi.group())
        for ci in iter_c_values:
            c_starts.append(ci.start())
            c_ends.append(ci.end())
            c_groups.append(ci.group())

        if (len(f_groups) == len(c_groups)):
            severities = mismatch_severities(f_groups, c_groups, config)
            if log_file is not None:
                num_format = "    <div><pre><code>{0:0%dd}"%num_len
                log_line = (num_format + ": ").format(line_num)
                log_line = log_line + c_line[0:c_starts[0]]
            for si in range(len(severities) - 1):
                if (severities[si] == 1): noisy += 1
                elif (severities[si] == 2): warnings += 1
                elif (severities[si] == 3): errors += 1
                if log_file is not None:
                    if (severities[si] == 0):
                        log_line = log_line + c_groups[si] + c_line[c_ends[si]:c_starts[si + 1]]
                    elif (severities[si] == 1):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
                    elif (severities[si] == 2):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,0,255)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
                    elif (severities[si] == 3):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
            if (len(severities) > 0):
                # Single errror test modification
                if (severities[-1] == 1): noisy += 1
                elif (severities[-1] == 2): warnings += 1
                elif (severities[-1] == 3):
                    split_c_line = c_line.split('/')
                    if (len(split_c_line) != 2): errors += 1
            if log_file is not None:
                if (len(severities) > 0):
                    if (severities[-1] == 0):
                        log_line = (
                            log_line + c_groups[-1] + c_line[c_ends[-1]:len(c_line) - 2]
                        )
                    elif (severities[-1] == 1):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                            + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                        )
                    elif (severities[-1] == 2):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,0,255)\"><code>"
                            + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                        )
                    elif (severities[-1] == 3):
                        split_c_line = c_line.split('/')
                        if (len(split_c_line) == 2):
                            log_line = (
                                log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                                + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                            )
                        else:
                            log_line = (
                                log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                                + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                            )
                log_file.write(log_line + "</code></pre></div>\n")
                log_file.write(ref_line)
        else: # The two lines contain a different number of numeric values
            if (log_file is not None):
                num_format = "    <div><pre><code>{0:0%dd}"%num_len
                log_line = (num_format + ": ").format(line_num)
                log_line = (
                    log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                    + c_line + "</code></span><code>"
                )
                log_file.write(log_line + "</code></pre></div>\n")
                log_file.write(ref_line)
            errors += 1
    return (errors, warnings, noisy)

## \brief Determine the severity of a numerical mismatch.
#
#  The severity scale is currently designed with the following integer codes:
#
#  0 - the values are equal
#
#  1 - the values are subject to suspect numerical noise (green fonts)
#
#  2 - the values are different but below error threshold (blue fonts)
#
#  3 - the values differ more than error threshold (red fonts)
#
#  \param str_f_values: `array(string)` The strings representing the numeric
#     values read from the FORTRAN output file.
#  \param str_c_values: `array(string)` The strings representing the numeric
#     values read from the C++ output file.
#  \param config: `dict` A dictionary containing the configuration options from
#     which to read the warning and the error threshold.
#
#  \returns result: `array(int)` An array of severity codes ordered as the
#  input numeric values.
def mismatch_severities(str_f_values, str_c_values, config):
    result = [0 for ri in range(len(str_f_values))]
    for i in range(len(str_f_values)):
        if (str_f_values[i] != str_c_values[i]):
            f_values = [float(str_f_values[j]) for j in range(len(str_f_values))]
            c_values = [float(str_c_values[j]) for j in range(len(str_c_values))]
            if (len(f_values) != len(c_values)): return []
            f_log_values = [0.0 for j in range(len(f_values))]
            c_log_values = [0.0 for j in range(len(c_values))]
            max_f_log = -1.0e12
            max_c_log = -1.0e12
            for j in range(len(f_values)) :
                if f_values[j] < 0.0: f_values[j] *= -1.0
                if c_values[j] < 0.0: c_values[j] *= -1.0
                f_log_values[j] = log10(f_values[j]) if f_values[j] > 0.0 else -999
                c_log_values[j] = log10(c_values[j]) if c_values[j] > 0.0 else -999
                if (f_log_values[j] > max_f_log): max_f_log = f_log_values[j]
                if (c_log_values[j] > max_c_log): max_c_log = f_log_values[j]
            if (c_log_values[i] < max_c_log - 5.0 and f_log_values[i] < max_f_log - 5.0):
                result[i] = 1
            else:
                difference = c_values[i] - f_values[i]
                fractional = 1.0
                if (f_values[i] != 0.0):
                    fractional = difference / f_values[i]
                if (fractional < 0.0): fractional *= -1.0
                if (fractional < config['warning_threshold']): result[i] = 2
                else: result[i] = 3
    return result
    
## \brief Parse the command line arguments.
#
#  The script behaviour can be modified through a set of mandatory and optional
#  arguments. Mandatory arguments are those required to execute a meaningful
#  comparison and they are limited to the names of the files that need to be
#  compared. The other arguments affect whether the script should produce an
#  HTML log file and what level of detail needs to be included in this log.
#
#  \returns config: `dict` A dictionary containing the script configuration.
def parse_arguments():
    config = {
        'fortran_file_name': '',
        'c_file_name': '',
        'full_log': False,
        'log_html': False,
        'html_output': 'pycompare.html',
        'warning_threshold': 0.005,
        'help_mode': False,
        'check_all': True,
    }
    for arg in argv[1:]:
        split_arg = arg.split("=")
        if (arg.startswith("--ffile")):
            config['fortran_file_name'] = split_arg[1]
        elif (arg.startswith("--cfile")):
            config['c_file_name'] = split_arg[1]
        elif (arg.startswith("--full")):
            config['full_log'] = True
        elif (arg.startswith("--html")):
            config['log_html'] = True
            if (len(split_arg) == 2):
                config['html_output'] = split_arg[1]
        elif (arg.startswith("--warn")):
            config['warning_threshold'] = float(split_arg[1])
        elif (arg.startswith("--help")):
            config['help_mode'] = True
        elif (arg.startswith("--quick")):
            config['check_all'] = False
        else:
            raise Exception("Unrecognized argument \'{0:s}\'".format(arg))
    return config

## \brief Print a command-line help summary.
def print_help():
    print("                                            ")
    print("***              PYCOMPARE               ***")
    print("                                            ")
    print("Compare the output of C++ and FORTRAN codes.")
    print("                                            ")
    print("Usage: \"./pycompare.py OPTIONS\"           ")
    print("                                            ")
    print("Valid options are:                          ")
    print("--ffile=FORTRAN_OUTPUT    File containing the output of the FORTRAN code (mandatory).")
    print("--cfile=C++_OUTPUT        File containing the output of the C++ code (mandatory).")
    print("--full                    Print all lines to log file (default prints only mismatches).")
    print("--help                    Print this help and exit.")
    print("--html[=OPT_OUTPUT_NAME]  Enable logging to HTML file (default logs to \"pycompare.html\").")
    print("--quick                   Stop on first mismatch (default is to perform a full check).")
    print("--warn                    Set a fractional threshold for numeric warning (default=0.005).")
    print("                                            ")

## \brief Add summary information to the HTML log file
#
#  In the case when a HTML log is requested, it is useful to obtain an overview
#  of the detected inconsistencies. This function undertakes the task of adding
#  a summary of the error, warning and noise counts on top of the log.
#  
#  \param config: `dict` A dictionary containing the script configuration.
#  \param errors: `int` The number of errors detected by the comparison.
#  \param warnings: `int` The number of warnings detected by the comparison.
#  \param noisy: `int` The number of noisy values detected by the comparison.
def reformat_log(config, errors, warnings, noisy):
    log_file = open(config['html_output'], 'r')
    log_lines = log_file.readlines()
    log_file.close()
    log_file = open(config['html_output'], 'w')
    for i in range(7): log_file.write(log_lines[i] + "\n")
    str_errors = "error" if errors == 1 else "errors"
    str_warnings = "warning" if warnings == 1 else "warnings"
    str_noisy = "noisy value" if noisy == 1 else "noisy values"
    summary = "    <div>Comparison yielded %d %s"%(errors, str_errors)
    summary = summary + ", %d %s"%(warnings, str_warnings)
    summary = summary + " and %d %s.</div>\n"%(noisy, str_noisy)
    log_file.write(summary)
    for i in range(7, len(log_lines)): log_file.write(log_lines[i] + "\n")
    log_file.close()
    

# ### PROGRAM EXECUTION ###
## \cond
res = main()
## \endcond
if (res > 0): exit(1)
exit(0)
