diff --git a/src/scripts/pycompare.py b/src/scripts/pycompare.py index 9831bd193fd653e633dd056bcdbe4b2fc57c62e7..bc22a698ece17b0ca0f6df5c8dd2e2165b33fff5 100755 --- a/src/scripts/pycompare.py +++ b/src/scripts/pycompare.py @@ -1,7 +1,7 @@ #!/bin/python -## @file pycompare -# Script to perform output consistency tests +## @package pycompare +# \brief Script to perform output consistency tests # # Comparing the numeric output can be rendered hard by the amount of information # contained in a typical output file and the necessity to determine whether a @@ -10,6 +10,11 @@ # the assumption that they were written by the FORTRAN and the C++ versions of # the code and to flag all the possible inconsistencies according to various # severity levels (namely: NOISE, WARNING, and ERROR). +# +# After execution, the script returns an exit code, which is set to 0, if no +# error-level inconsistencies were found, or 1 otherwise. This can be used by +# subsequent system calls to set up a testing suite checking whether the code +# is able to reproduce legacy results. import re @@ -54,6 +59,35 @@ def main(): return errors ## \brief Perform the comparison of two files. +# +# The comparison is executed as a line-by-line process. In order to process +# files correctly, it is required that the two input files have exactly the +# same format (with the exception of some number formatting subtleties that +# are handled by regular expressions). Therefore, the first comparison step +# is testing whether the input files have the same number of lines. If this +# condition is not met, a formatting problem is assumed and every line in +# the C++ output file is counted as an error. Otherwise, all the numeric +# values found in the result are compared for consistency within warning +# tolerance. Warnings and errors are issued if two values do not match by a +# fractional amount being, respectively, below or above the threshold for +# warning. A special case is the detection of suspect numeric noise. This +# arises on very small quantities as a consequence of differences in the +# level of approximation or in the hardware implementation of the numeric +# values, which typically has negligible impact on the overall reslults, +# even though it can have potentially large fractional mismatches, because +# it is caused by values that are close to 0. +# +# Numeric noise is filtered by taking advantage from the fact that the +# output files are formatted in such a way that values with similar physical +# meaning are written to the same output line. If the comparison results in +# a large fractional mismatch on a value that is more than 5 orders of +# magnitude smaller than the highest order of magnitude that was read from +# the current row, the discrepancy is flagged as a potential noise effect. +# +# \param config: `dict` A dictionary containing the script configuration. +# +# \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles +# together the numbers of detected errors, warnings and noisy values. def compare_files(config): mismatch_count = { 'errors': 0, @@ -108,6 +142,21 @@ def compare_files(config): return mismatch_count ## \brief Perform the comparison of two file lines. +# +# This function handles the line-by-line comparison of coded result files. Depending +# on whether a HTML log report was requested, it also undertakes the task of +# formatting the HTML code to show the comparison results as high-lighted entries, +# according to the severity degree of the mismatch. +# +# \param f_line: `string` A line extracted from the FORTRAN output file. +# \param c_line: `string` A line extracted from the C++ output file. +# \param config: `dict` A dictionary containing the script configuration. +# \param line_num: `int` The number of the current line (0-indexed). +# \param num_len: `int` The number digits to format the line number tag in the HTML log. +# \param log_file: `file` A file where to write logging information, if required. +# +# \returns mismatch_count: `tuple(int, int, int)` A tuple that bundles +# together the numbers of detected errors, warnings and noisy values. def compare_lines(f_line, c_line, config, line_num=0, num_len=1, log_file=None): errors = 0 warnings = 0 @@ -204,9 +253,13 @@ def compare_lines(f_line, c_line, config, line_num=0, num_len=1, log_file=None): ## \brief Determine the severity of a numerical mismatch. # # The severity scale is currently designed with the following integer codes: +# # 0 - the values are equal +# # 1 - the values are subject to suspect numerical noise (green fonts) +# # 2 - the values are different but below error threshold (blue fonts) +# # 3 - the values differ more than error threshold (red fonts) # # \param str_f_values: `array(string)` The strings representing the numeric @@ -215,6 +268,9 @@ def compare_lines(f_line, c_line, config, line_num=0, num_len=1, log_file=None): # values read from the C++ output file. # \param config: `dict` A dictionary containing the configuration options from # which to read the warning and the error threshold. +# +# \returns result: `array(int)` An array of severity codes ordered as the +# input numeric values. def mismatch_severities(str_f_values, str_c_values, config): result = [0 for ri in range(len(str_f_values))] for i in range(len(str_f_values)): @@ -246,6 +302,14 @@ def mismatch_severities(str_f_values, str_c_values, config): return result ## \brief Parse the command line arguments. +# +# The script behaviour can be modified through a set of mandatory and optional +# arguments. Mandatory arguments are those required to execute a meaningful +# comparison and they are limited to the names of the files that need to be +# compared. The other arguments affect whether the script should produce an +# HTML log file and what level of detail needs to be included in this log. +# +# \returns config: `dict` A dictionary containing the script configuration. def parse_arguments(): config = { 'fortran_file_name': '', @@ -298,6 +362,12 @@ def print_help(): print("--warn Set a fractional threshold for numeric warning (default=0.005).") print(" ") +## \brief Add summary information to the HTML log file +# +# \param config: `dict` A dictionary containing the script configuration. +# \param errors: `int` The number of errors detected by the comparison. +# \param warnings: `int` The number of warnings detected by the comparison. +# \param noisy: `int` The number of noisy values detected by the comparison. def reformat_log(config, errors, warnings, noisy): log_file = open(config['html_output'], 'r') log_lines = log_file.readlines()