cantera/test/SConscript

import os
import subprocess
from xml.etree import ElementTree
from os.path import join as pjoin
import time
import sys

from buildutils import *

Import('env','build','install')
localenv = env.Clone()

localenv.Prepend(CPPPATH=['#include'],
                 LIBPATH='#build/lib')
localenv.Append(LIBS=localenv['cantera_shared_libs'],
                CCFLAGS=env['warning_flags'])

if env['python_package'] != 'full':
    localenv.Append(CPPDEFINES={'CT_SKIP_PYTHON': '1'})

if env['googletest'] == 'submodule':
    localenv.Prepend(CPPPATH=['#ext/googletest/googletest/include',
                              '#ext/googletest/googlemock/include'])
if env['googletest'] != 'none':
    localenv.Append(LIBS=['gtest', 'gmock'])

# Turn off optimization to speed up compilation
ccflags = localenv['CCFLAGS']
for optimize_flag in ('-O3', '-O2', '/O2'):
    if optimize_flag in ccflags:
        ccflags.remove(optimize_flag)
localenv['CCFLAGS'] = ccflags

localenv['ENV']['CANTERA_DATA'] = (Dir('#data').abspath + os.pathsep +
                                   Dir('#samples/data').abspath + os.pathsep +
                                   Dir('#test/data').abspath)

# For Python model extensions
localenv.PrependENVPath('PYTHONPATH', Dir('#test/python').abspath)

PASSED_FILES = {}

def addTestProgram(subdir, progName, env_vars={}):
    """
    Compile a test program and create a targets for running
    and resetting the test.
    """
    def gtestRunner(target, source, env):
        """SCons Action to run a compiled gtest program"""
        program = source[0]
        passedFile = target[0]
        test_results.tests.pop(passedFile.name, None)
        workDir = Dir('#test/work').abspath
        resultsFile = pjoin(workDir, 'gtest-%s.xml' % progName)
        if os.path.exists(resultsFile):
            os.remove(resultsFile)

        if not os.path.isdir(workDir):
            os.mkdir(workDir)
        cmd = [program.abspath, '--gtest_output=xml:'+resultsFile]
        cmd.extend(env['gtest_flags'].split())
        if env["fast_fail_tests"]:
            env["ENV"]["GTEST_BREAK_ON_FAILURE"] = "1"
        code = subprocess.call(cmd, env=env['ENV'], cwd=workDir)

        if code:
            logger.failed(f"Test {progName!r} exited with code {code}")
            if env["fast_fail_tests"]:
                sys.exit(1)
        else:
            # Test was successful
            with open(passedFile.path, 'w') as passed_file:
                passed_file.write(time.asctime()+'\n')

        if os.path.exists(resultsFile):
            # Determine individual test status
            results = ElementTree.parse(resultsFile)
            for test in results.findall('.//testcase'):
                testName = '{0}: {1}.{2}'.format(progName, test.get('classname'),
                                                 test.get('name'))
                if test.findall('failure'):
                    test_results.failed[testName] = 1
                else:
                    test_results.passed[testName] = 1
        else:
            # Total failure of the test program - unable to determine status of
            # individual tests. This is potentially very bad, so it counts as
            # more than one failure.
            test_results.failed[passedFile.name +
                ' ***no results for entire test suite***'] = 100

    testenv = localenv.Clone()
    testenv['ENV'].update(env_vars)
    if env["skip_slow_tests"]:
        testenv.Append(CPPDEFINES=["CT_SKIP_SLOW"])
    if env["python_package"] == "none":
        testenv.Append(CPPDEFINES=["CT_NO_PYTHON"])
    program = testenv.Program(pjoin(subdir, progName),
                              multi_glob(testenv, subdir, 'cpp'))
    passedFile = File(pjoin(str(program[0].dir), '%s.passed' % program[0].name))
    PASSED_FILES[progName] = str(passedFile)
    test_results.tests[passedFile.name] = program
    if env['googletest'] != 'none':
        run_program = testenv.Command(passedFile, program, gtestRunner)
        env.Depends(run_program, env['build_targets'])
        env.Depends(env['test_results'], run_program)
        Alias(f'test-{progName}', run_program)
        Alias('test-gtest', run_program)
        Alias(f'build-{progName}', program)
        Alias('build-tests', program)
        env['testNames'].append(progName)
    else:
        test_results.failed['test-%s (googletest disabled)' % progName] = 1

    if os.path.exists(passedFile.abspath):
        Alias('test-reset', testenv.Command('reset-%s%s' % (subdir, progName),
                                            [], [Delete(passedFile.abspath)]))


def addPythonTest(testname, subset):
    """
    Create targets for running and resetting a test script.
    """

    def scriptRunner(target, source, env):
        """Scons Action to run a set of tests using pytest """
        pytest_outfile = File("#test/work/pytest.xml").abspath
        workDir = Dir('#test/work').abspath
        passedFile = target[0]
        test_results.tests.pop(passedFile.name, None)
        if not os.path.isdir(workDir):
            os.mkdir(workDir)
        if os.path.exists(pytest_outfile):
            os.remove(pytest_outfile)

        environ = dict(env['ENV'])
        if env["skip_slow_tests"]:
            environ["CT_SKIP_SLOW"] = "1"

        cmd = [env.subst("$python_cmd"), "-m", "pytest", "-raP",
               "--junitxml=test/work/pytest.xml"]
        if env["fast_fail_tests"]:
            cmd.append("-x")
        if env["show_long_tests"]:
            cmd.append("--durations=50")

        if env["verbose_tests"]:
            cmd.append("-v")
        else:
            cmd.append("--log-level=ERROR")

        if env["coverage"]:
            cmd.extend([
                "--cov=cantera",
                "--cov-config=test/python/coverage.ini",
                "--cov-report=xml:build/pycov.xml",
                "--cov-report=html:build/python-coverage"
            ])

        code = subprocess.call(cmd + [str(s) for s in source], env=environ)

        if not code:
            # Test was successful
            with open(target[0].path, 'w') as passed_file:
                passed_file.write(time.asctime()+'\n')
        elif env["fast_fail_tests"]:
            sys.exit(1)

        failures = 0
        if os.path.exists(pytest_outfile):
            results = ElementTree.parse(pytest_outfile)
            for test in results.findall('.//testcase'):
                class_name = test.get('classname')
                if class_name.startswith("build.python.cantera.test."):
                    class_name = class_name[26:]
                test_name = "python: {}.{}".format(class_name, test.get('name'))
                if test.findall('failure'):
                    test_results.failed[test_name] = 1
                    failures += 1
                else:
                    test_results.passed[test_name] = 1

        if code and failures == 0:
            # Failure, but unable to determine status of individual tests. This
            # gets counted as many failures.
            test_results.failed[testname +
                ' ***no results for entire test suite***'] = 100

    testenv = localenv.Clone()
    passedFile = File(f'python/{name}.passed')
    PASSED_FILES[testname] = str(passedFile)

    if subset:
        test_files = [f"python/test_{subset}.py"]
    else:
        test_files = multi_glob(localenv, "python", "^test_*.py")
    run_program = testenv.Command(passedFile, test_files, scriptRunner)
    testenv.Depends(run_program, localenv["python_module"])
    testenv.Depends(run_program, localenv["python_extension"])
    if not subset:
        testenv.Depends(env['test_results'], run_program)
        test_results.tests[passedFile.name] = True
    if os.path.exists(passedFile.abspath):
        Alias("test-reset", testenv.Command(f"reset-python{testname}",
                                            [], [Delete(passedFile.abspath)]))

    return run_program


# Instantiate tests
addTestProgram('clib', 'clib')
addTestProgram('equil', 'equil')
addTestProgram('general', 'general')
addTestProgram('kinetics', 'kinetics')
addTestProgram('oneD', 'oneD')
addTestProgram('thermo', 'thermo')
addTestProgram('thermo_consistency', 'thermo-consistency',
               env_vars={'GTEST_BRIEF': '0' if env['verbose_tests'] else '1'})
addTestProgram('transport', 'transport')
addTestProgram('zeroD', 'zeroD')

python_subtests = ['']
test_root = '#test/python'
for f in multi_glob(localenv, test_root, '^test_*.py'):
    python_subtests.append(f.name[5:-3])

if localenv['python_package'] == 'full':
    # Create test aliases for individual test modules (such as test-python-thermo;
    # not run as part of the main suite) and a single test runner with all the
    # tests (test-python) for the main suite.
    for subset in python_subtests:
        name = 'python-' + subset if subset else 'python'
        pyTest = addPythonTest(name, subset=subset)
        localenv.Alias('test-' + name, pyTest)
        env['testNames'].append(name)

# Force explicitly-named tests to run even if SCons thinks they're up to date
for command in COMMAND_LINE_TARGETS:
    if command.startswith('test-'):
        name = command[5:]
        if name in PASSED_FILES and os.path.exists(PASSED_FILES[name]):
            os.remove(PASSED_FILES[name])