#!/bin/python3

## @package pycompare
#  \brief Script to perform output consistency tests
#
#  Comparing the numeric output can be rendered hard by the amount of information
#  contained in a typical output file and the necessity to determine whether a
#  difference is actually significant or just caused by numeric noise hitting
#  negligible values. The task of `pycompare.py` is to compare two output files, in
#  the assumption that they were written by the FORTRAN and the C++ versions of
#  the code and to flag all the possible inconsistencies according to various
#  severity levels (namely: NOISE, WARNING, and ERROR).
#
#  After execution, the script returns an exit code, which is set to 0, if no
#  error-level inconsistencies were found, or 1 otherwise. This can be used by
#  subsequent system calls to set up a testing suite checking whether the code
#  is able to reproduce legacy results.
#
#  The script execution requires python3.

import re
import os

from math import log10
from sys import argv

## \cond
number_reg = re.compile(r'-?[0-9]\.[0-9]+E?[-+][0-9]{2,5}')
## \endcond

## \brief Main execution code
#
# `main()` is the function that handles the creation of the script configuration
# and the execution of the comparison. It returns an integer value corresponding
# to the number of detected error-level inconsistencies.
#
# \returns errors: `int` Number of detected error-level inconsistencies.
def main():
    config = parse_arguments()
    errors, warnings, noisy = (0, 0, 0)
    if config['help_mode'] or len(argv) == 1:
        config['help_mode'] = True
        print_help()
    else:
        compare_log = compare_files(config)
        errors = compare_log['errors']
        warnings = compare_log['warnings']
        noisy = compare_log['noisy']
        print("ERROR COUNT: %d"%errors)
        print("WARNING COUNT: %d"%warnings)
        print("NOISE COUNT: %d"%noisy)
        if (config['log_html']): reformat_log(config, errors, warnings, noisy)
    if (errors > 0):
        print("FAILURE: {0:s} is not consistent with {1:s}".format(
            config['c_file_name'], config['fortran_file_name']
        ))
    else:
        if (not config['help_mode']):
            print("SUCCESS: {0:s} is consistent with {1:s}".format(
                config['c_file_name'], config['fortran_file_name']
            ))
    return errors

## \brief Perform the comparison of two files.
#
#  The comparison is executed as a line-by-line process. In order to process
#  files correctly, it is required that the two input files have exactly the
#  same format (with the exception of some number formatting subtleties that
#  are handled by regular expressions). Therefore, the first comparison step
#  is testing whether the input files have the same number of lines. If this
#  condition is not met, a formatting problem is assumed and every line in
#  the C++ output file is counted as an error. Otherwise, all the numeric
#  values found in the result are compared for consistency within warning
#  tolerance. Warnings and errors are issued if two values do not match by a
#  fractional amount being, respectively, below or above the threshold for
#  warning. A special case is the detection of suspect numeric noise. This
#  arises on very small quantities as a consequence of differences in the
#  level of approximation or in the hardware implementation of the numeric
#  values, which typically has negligible impact on the overall reslults,
#  even though it can have potentially large fractional mismatches, because
#  it is caused by values that are close to 0.
#
#  Numeric noise is filtered by taking advantage from the fact that the
#  output files are formatted in such a way that values with similar physical
#  meaning are written to the same output line. If the comparison results in
#  a large fractional mismatch on a value that is more than 5 orders of
#  magnitude smaller than the highest order of magnitude that was read from
#  the current row, the discrepancy is flagged as a potential noise effect.
#
#  \param config: `dict` A dictionary containing the script configuration.
#
#  \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles
#  together the numbers of detected errors, warnings and noisy values.
def compare_files(config):
    mismatch_count = {
        'errors': 0,
        'warnings': 0,
        'noisy': 0,
    }
    fortran_file = open(config['fortran_file_name'], 'r')
    c_file = open(config['c_file_name'], 'r')
    l_file = None
    f_lines = []
    c_lines = []
    line_count = 0
    if (not config['linewise']):
        f_lines = fortran_file.readlines()
        c_lines = c_file.readlines()
        line_count = len(f_lines)
        fortran_file.close()
        c_file.close()
    else: # line-wise comparison mode
        f_lines = [fortran_file.readline()]
        c_lines = [c_file.readline()]
        print("INFO: using line-wise mode")
        print("INFO: counting result lines...")
        while (f_lines[0] != ''):
            if (c_lines[0].startswith("INSERTION:")):
                c_lines = [c_file.readline()]
            if (c_lines[0] != ''):
                line_count += 1
            else:
                print("ERROR: C++ file is shorter than FORTRAN file.")
                fortran_file.close()
                c_file.close()
                mismatch_count['errors'] = line_count
                return mismatch_count
            f_lines[0] = fortran_file.readline()
            c_lines[0] = c_file.readline()
        if (c_lines[0] not in ['', '\n']):
            print("ERROR: C++ file is longer than FORTRAN file.")
            fortran_file.close()
            c_file.close()
            mismatch_count['errors'] = line_count
            return mismatch_count
        fortran_file.close()
        c_file.close()
        print("INFO: the output files have %d lines"%line_count)
        fortran_file = open(config['fortran_file_name'], 'r')
        c_file = open(config['c_file_name'], 'r')
    num_read_lines = 0
    # LOG FILE INITIALIZATION #
    if (config['log_html']):
        l_file = open(config['html_output'], 'w')
        l_file.write("<!DOCTYPE html>\n")
        l_file.write("<html xmnls=\"http://www.w3.org/1999/xhtml\">\n")
        l_file.write("  <header>\n")
        l_file.write(
            "    <h1>Comparison between {0:s} and {1:s}</h1>\n".format(
                config['fortran_file_name'], config['c_file_name']
            )
        )
        l_file.write("  </header>\n")
        l_file.write("  <body>\n")
        l_file.write("    <div>Numeric noise is marked <span style=\"font-weight: bold; color: rgb(0,185,0)\">"
                     + "GREEN</span>, warnings are marked <span style=\"font-weight: bold; color: rgb(0,0,255)\">"
                     + "BLUE</span> and errors are marked <span style=\"font-weight: bold; color: rgb(255,0,0)\">"
                     + "RED</span>.</div>\n")
    # END LOG FILE INITIALIZATION #
    line_loop = True
    num_len = 1
    if (line_count > 0):
        num_len = max(4, int(log10(line_count)) + 1)
    print("INFO: checking file contents...")
    while (line_loop):
        if (not config['linewise']):
            line_loop = False
        else:
            f_lines = [fortran_file.readline()]
            c_lines = [c_file.readline()]
            if (c_lines[0].startswith("INSERTION:")):
                c_lines = [c_file.readline()]
                num_read_lines += 1
            num_read_lines += 1
        # Start here the comparison loop
        if (len(f_lines) == len(c_lines)):
            for li in range(len(f_lines)):
                line_result = compare_lines(f_lines[li], c_lines[li], config, num_read_lines, num_len, l_file)
                mismatch_count['errors'] += line_result[0]
                mismatch_count['warnings'] += line_result[1]
                mismatch_count['noisy'] += line_result[2]
                if (mismatch_count['errors'] > 0 and not config['check_all']):
                    print("INFO: mismatch found at line %d"%(num_read_lines))
                    line_loop = False
                    break
        else:
            mismatch_count['errors'] = len(c_lines)
            print("ERROR: {0:s} and {1:s} have different numbers of lines!".format(
                config['fortran_file_name'], config['c_file_name']
            ))
            if (config['log_html']):
                print("Different file sizes. No log produced.")
                config['log_html'] = False
        if (num_read_lines >= line_count):
            line_loop = False
        #End line loop
    if l_file is not None:
        l_file.write("  </body>\n")
        l_file.write("</html>\n")
        l_file.close()
    return mismatch_count

## \brief Perform the comparison of two file lines.
#
#  This function handles the line-by-line comparison of coded result files. Depending
#  on whether a HTML log report was requested, it also undertakes the task of
#  formatting the HTML code to show the comparison results as high-lighted entries,
#  according to the severity degree of the mismatch.
#
#  \param f_line: `string` A line extracted from the FORTRAN output file.
#  \param c_line: `string` A line extracted from the C++ output file.
#  \param config: `dict` A dictionary containing the script configuration.
#  \param line_num: `int` The number of the current line (0-indexed).
#  \param num_len: `int` The number digits to format the line number tag in the HTML log.
#  \param log_file: `file` A file where to write logging information, if required.
#
#  \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles
#  together the numbers of detected errors, warnings and noisy values.
def compare_lines(f_line, c_line, config, line_num=0, num_len=4, log_file=None):
    errors = 0
    warnings = 0
    noisy = 0
    f_line = f_line.replace("D-","E-").replace("D+","E+")
    ref_format = "    <div><span style=\"font-weight: bold; color: rgb(125,125,125)\"><pre><code>{0:%ds}"%num_len
    ref_line = (ref_format + ": {1:s}</code></pre></span></div>\n").format("ORIG", f_line[:-1])
    log_line = ""
    if (f_line == c_line):
        if log_file is not None:
            if (config['full_log']):
                num_format = "    <div><pre><code>{0:0%dd}"%num_len
                log_line = (num_format + ": {1:s}</code></pre></div>\n").format(line_num, c_line[:-1])
                log_file.write(log_line)
    else:
        iter_f_values = number_reg.finditer(f_line)
        iter_c_values = number_reg.finditer(c_line)
        f_starts, f_ends, f_groups = [], [], []
        c_starts, c_ends, c_groups = [], [], []
        for fi in iter_f_values:
            f_starts.append(fi.start())
            f_ends.append(fi.end())
            f_groups.append(fi.group())
        for ci in iter_c_values:
            c_starts.append(ci.start())
            c_ends.append(ci.end())
            c_groups.append(ci.group())

        if (len(f_groups) == len(c_groups)):
            severities = mismatch_severities(f_groups, c_groups, config)
            if log_file is not None:
                if (len(severities) > 0):
                    num_format = "    <div><pre><code>{0:0%dd}"%num_len
                    log_line = (num_format + ": ").format(line_num)
                    log_line = log_line + c_line[0:c_starts[0]]
            for si in range(len(severities) - 1):
                if (severities[si] == 1): noisy += 1
                elif (severities[si] == 2): warnings += 1
                elif (severities[si] == 3): errors += 1
                if log_file is not None:
                    if (severities[si] == 0):
                        log_line = log_line + c_groups[si] + c_line[c_ends[si]:c_starts[si + 1]]
                    elif (severities[si] == 1):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
                    elif (severities[si] == 2):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,0,255)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
                    elif (severities[si] == 3):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                            + c_groups[si] + "</code></span><code>" + c_line[c_ends[si]:c_starts[si + 1]]
                        )
            if (len(severities) > 0):
                # Single error test modification
                if (severities[-1] == 1): noisy += 1
                elif (severities[-1] == 2): warnings += 1
                elif (severities[-1] == 3):
                    split_c_line = c_line.split('/')
                    if (config['warning_threshold'] == 0.0): errors += 1
                    elif (len(split_c_line) != 2): errors += 1
            if log_file is not None:
                if (len(severities) > 0):
                    if (severities[-1] == 0):
                        log_line = (
                            log_line + c_groups[-1] + c_line[c_ends[-1]:len(c_line) - 1]
                        )
                    if (severities[-1] == 1):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                            + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                        )
                    if (severities[-1] == 2):
                        log_line = (
                            log_line + "</code><span style=\"font-weight: bold; color: rgb(0,0,255)\"><code>"
                            + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                        )
                    if (severities[-1] == 3):
                        split_c_line = c_line.split('/')
                        if (len(split_c_line) == 2):
                            if (config['warning_threshold'] != 0.0):
                                log_line = (
                                    log_line + "</code><span style=\"font-weight: bold; color: rgb(0,185,0)\"><code>"
                                    + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                                )
                            else:
                                log_line = (
                                    log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                                    + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                                )
                        else:
                            log_line = (
                                log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                                + c_groups[-1] + "</code></span><code>" + c_line[c_ends[-1]:len(c_line) - 2]
                            )
                log_file.write(log_line + "</code></pre></div>\n")
                log_file.write(ref_line)
        else: # The two lines contain a different number of numeric values
            if (log_file is not None):
                num_format = "    <div><pre><code>{0:0%dd}"%num_len
                log_line = (num_format + ": ").format(line_num)
                log_line = (
                    log_line + "</code><span style=\"font-weight: bold; color: rgb(255,0,0)\"><code>"
                    + c_line + "</code></span><code>"
                )
                log_file.write(log_line + "</code></pre></div>\n")
                log_file.write(ref_line)
            errors += 1
    return (errors, warnings, noisy)

## \brief Determine the severity of a numerical mismatch.
#
#  The severity scale is currently designed with the following integer codes:
#
#  0 - the values are equal
#
#  1 - the values are subject to suspect numerical noise (green fonts)
#
#  2 - the values are different but below error threshold (blue fonts)
#
#  3 - the values differ more than error threshold (red fonts)
#
#  \param str_f_values: `array(string)` The strings representing the numeric
#     values read from the FORTRAN output file.
#  \param str_c_values: `array(string)` The strings representing the numeric
#     values read from the C++ output file.
#  \param config: `dict` A dictionary containing the configuration options from
#     which to read the warning and the error threshold.
#
#  \returns result: `array(int)` An array of severity codes ordered as the
#  input numeric values.
def mismatch_severities(str_f_values, str_c_values, config):
    result = []
    if len(str_f_values) == len(str_c_values):
        result = [0 for ri in range(len(str_f_values))]
        f_values = []
        c_values = []
        # Convert numeric strings to numbers
        for i in range(len(str_f_values)):
            # Add the exponent marker if it is missing
            temp_str_value = str_f_values[i][1:]
            split_temp = temp_str_value.split('-')
            if len(split_temp) > 1:
                if (split_temp[0][-1] != 'E'):
                    str_f_values[i] = str_f_values[i][0] + split_temp[0] + "E-" + split_temp[1]
            f_values.append(float(str_f_values[i]))
            temp_str_value = str_c_values[i][1:]
            split_temp = temp_str_value.split('-')
            if len(split_temp) > 1:
                if (split_temp[0][-1] != 'E'):
                    str_c_values[i] = str_c_values[i][0] + split_temp[0] + "E-" + split_temp[1]
            c_values.append(float(str_c_values[i]))
            # End of missing exponent marker correction
        # End string to number conversion
        # Evaluate the maximum scale
        max_f_log = -1.0e12
        max_c_log = -1.0e12
        for si in range(len(f_values)):
            if (f_values[si] != 0):
                sign = 1.0 if f_values[si] > 0.0 else -1.0
                log_f_value = log10(sign * f_values[si])
                if (log_f_value > max_f_log): max_f_log = log_f_value
            if (c_values[si] != 0):
                sign = 1.0 if c_values[si] > 0.0 else -1.0
                log_c_value = log10(sign * c_values[si])
                if (log_c_value > max_c_log): max_c_log = log_c_value
        if (max_f_log == -1.0e12): max_f_log = 0.0
        if (max_c_log == -1.0e12): max_c_log = 0.0
        # End of maximum scale evaluation
        # Compare the numbers
        for i in range(len(f_values)):
            if (f_values[i] != c_values[i]):
                if (f_values[i] != 0.0):
                    sign = 1.0 if f_values[i] > 0.0 else -1.0
                    log_f_value = log10(sign * f_values[i])
                    if (log_f_value > max_f_log - 5.0):
                        scale = 10.0**(log_f_value - max_f_log)
                        fractional = scale * (f_values[i] - c_values[i]) / f_values[i]
                        if (fractional < 0.0): fractional *= -1.0
                        if (fractional <= config['warning_threshold']):
                            result[i] = 2
                        else:
                            result[i] = 3
                    else:
                        result[i] = 1
                else: # f_values[i] == 0 and c_values[i] != 0
                    sign = 1.0 if c_values[i] > 0.0 else -1.0
                    log_c_value = log10(sign * c_values[i])
                    if (log_c_value > max_c_log - 5.0):
                        scale = 10.0**(log_c_value - max_c_log)
                        fractional = scale * (c_values[i] - f_values[i]) / c_values[i]
                        if (fractional < 0.0): fractional *= -1.0
                        if (fractional <= config['warning_threshold']):
                            result[i] = 2
                        else:
                            result[i] = 3
                    else:
                        result[i] = 1
        # End number comparison
    return result
    
## \brief Parse the command line arguments.
#
#  The script behaviour can be modified through a set of mandatory and optional
#  arguments. Mandatory arguments are those required to execute a meaningful
#  comparison and they are limited to the names of the files that need to be
#  compared. The other arguments affect whether the script should produce an
#  HTML log file and what level of detail needs to be included in this log.
#
#  \returns config: `dict` A dictionary containing the script configuration.
def parse_arguments():
    config = {
        'fortran_file_name': '',
        'c_file_name': '',
        'full_log': False,
        'linewise': True,
        'log_html': False,
        'html_output': 'pycompare.html',
        'warning_threshold': 0.005,
        'help_mode': False,
        'check_all': True,
    }
    for arg in argv[1:]:
        split_arg = arg.split("=")
        if (arg.startswith("--ffile")):
            config['fortran_file_name'] = split_arg[1]
        elif (arg.startswith("--cfile")):
            config['c_file_name'] = split_arg[1]
        elif (arg.startswith("--full")):
            config['full_log'] = True
        elif (arg.startswith("--html")):
            config['log_html'] = True
            if (len(split_arg) == 2):
                config['html_output'] = split_arg[1]
        elif (arg.startswith("--warn")):
            config['warning_threshold'] = float(split_arg[1])
        elif (arg.startswith("--help")):
            config['help_mode'] = True
        elif (arg.startswith("--linewise")):
            config['linewise'] = True
        elif (arg.startswith("--quick")):
            config['check_all'] = False
        else:
            raise Exception("Unrecognized argument \'{0:s}\'".format(arg))
    return config

## \brief Print a command-line help summary.
def print_help():
    print("                                            ")
    print("***              PYCOMPARE               ***")
    print("                                            ")
    print("Compare the output of C++ and FORTRAN codes.")
    print("                                            ")
    print("Usage: \"./pycompare.py OPTIONS\"           ")
    print("                                            ")
    print("Valid options are:                          ")
    print("--ffile=FORTRAN_OUTPUT    File containing the output of the FORTRAN code (mandatory).")
    print("--cfile=C++_OUTPUT        File containing the output of the C++ code (mandatory).")
    print("--full                    Print all lines to log file (default prints only mismatches).")
    print("--help                    Print this help and exit.")
    print("--html[=OPT_OUTPUT_NAME]  Enable logging to HTML file (default logs to \"pycompare.html\").")
    print("--linewise                Load only one line at a time. Useful to compare big files (True by default).")
    print("--quick                   Stop on first mismatch (default is to perform a full check).")
    print("--warn                    Set a fractional threshold for numeric warning (default = 0.005).")
    print("                                            ")

## \brief Add summary information to the HTML log file
#
#  In the case when a HTML log is requested, it is useful to obtain an overview
#  of the detected inconsistencies. This function undertakes the task of adding
#  a summary of the error, warning and noise counts on top of the log.
#  
#  \param config: `dict` A dictionary containing the script configuration.
#  \param errors: `int` The number of errors detected by the comparison.
#  \param warnings: `int` The number of warnings detected by the comparison.
#  \param noisy: `int` The number of noisy values detected by the comparison.
def reformat_log(config, errors, warnings, noisy):
    log_file = open(config['html_output'], 'r')
    new_file = open("PYCOMPARE_TEMPORARY_LOG.html", 'w')
    for hi in range(7):
        log_line = log_file.readline()
        new_file.write(log_line)
    str_errors = "error" if errors == 1 else "errors"
    str_warnings = "warning" if warnings == 1 else "warnings"
    str_noisy = "noisy value" if noisy == 1 else "noisy values"
    summary = "    <div>Comparison yielded %d %s"%(errors, str_errors)
    summary = summary + ", %d %s"%(warnings, str_warnings)
    summary = summary + " and %d %s.</div>\n"%(noisy, str_noisy)
    new_file.write(summary)
    log_line = log_file.readline()
    while (log_line != ''):
        new_file.write(log_line)
        log_line = log_file.readline()
    log_file.close()
    new_file.close()
    os.remove(config['html_output'])
    os.rename("PYCOMPARE_TEMPORARY_LOG.html", config['html_output'])

# ### PROGRAM EXECUTION ###
## \cond
res = main()
## \endcond
if (res > 0): exit(1)
exit(0)
