cantera/test/SConscript

import os
import subprocess
from xml.etree import ElementTree
from os.path import join as pjoin
import time

from buildutils import *

Import('env','build','install')
localenv = env.Clone()

# Where possible, link tests against the shared libraries to minimize the sizes
# of the resulting binaries.
if localenv['OS'] == 'Linux':
    cantera_libs = localenv['cantera_shared_libs']
else:
    cantera_libs = localenv['cantera_libs']

localenv.Prepend(CPPPATH=['#include'],
                 LIBPATH='#build/lib')
localenv.Append(LIBS=cantera_libs,
                CCFLAGS=env['warning_flags'])

if env['googletest'] == 'submodule':
    localenv.Prepend(CPPPATH=['#ext/googletest/googletest/include',
                              '#ext/googletest/googlemock/include'])
if env['googletest'] != 'none':
    localenv.Append(LIBS=['gtest', 'gmock'])

# Turn off optimization to speed up compilation
ccflags = localenv['CCFLAGS']
for optimize_flag in ('-O3', '-O2', '/O2'):
    if optimize_flag in ccflags:
        ccflags.remove(optimize_flag)
localenv['CCFLAGS'] = ccflags

localenv['ENV']['CANTERA_DATA'] = (Dir('#build/data').abspath + os.pathsep +
                                   Dir('#test/data').abspath)

PASSED_FILES = {}

# Add build/lib in order to find Cantera shared library
localenv.PrependENVPath('LD_LIBRARY_PATH', Dir('#build/lib').abspath)

def addTestProgram(subdir, progName, env_vars={}):
    """
    Compile a test program and create a targets for running
    and resetting the test.
    """
    def gtestRunner(target, source, env):
        """SCons Action to run a compiled gtest program"""
        program = source[0]
        passedFile = target[0]
        test_results.tests.pop(passedFile.name, None)
        workDir = Dir('#test/work').abspath
        resultsFile = pjoin(workDir, 'gtest-%s.xml' % progName)
        if os.path.exists(resultsFile):
            os.remove(resultsFile)

        if not os.path.isdir(workDir):
            os.mkdir(workDir)
        cmd = [program.abspath, '--gtest_output=xml:'+resultsFile]
        cmd.extend(env['gtest_flags'].split())
        if env["fast_fail_tests"]:
            env["ENV"]["GTEST_BREAK_ON_FAILURE"] = "1"
        code = subprocess.call(cmd, env=env['ENV'], cwd=workDir)

        if code:
            print("FAILED: Test '{0}' exited with code {1}".format(progName, code))
            if env["fast_fail_tests"]:
                sys.exit(1)
        else:
            # Test was successful
            with open(passedFile.path, 'w') as passed_file:
                passed_file.write(time.asctime()+'\n')

        if os.path.exists(resultsFile):
            # Determine individual test status
            results = ElementTree.parse(resultsFile)
            for test in results.findall('.//testcase'):
                testName = '{0}: {1}.{2}'.format(progName, test.get('classname'),
                                                 test.get('name'))
                if test.findall('failure'):
                    test_results.failed[testName] = 1
                else:
                    test_results.passed[testName] = 1
        else:
            # Total failure of the test program - unable to determine status of
            # individual tests. This is potentially very bad, so it counts as
            # more than one failure.
            test_results.failed[passedFile.name +
                ' ***no results for entire test suite***'] = 100

    testenv = localenv.Clone()
    testenv['ENV'].update(env_vars)
    program = testenv.Program(pjoin(subdir, progName),
                              multi_glob(testenv, subdir, 'cpp'))
    passedFile = File(pjoin(str(program[0].dir), '%s.passed' % program[0].name))
    PASSED_FILES[progName] = str(passedFile)
    test_results.tests[passedFile.name] = program
    if env['googletest'] != 'none':
        run_program = testenv.Command(passedFile, program, gtestRunner)
        env.Depends(run_program, env['build_targets'])
        env.Depends(env['test_results'], run_program)
        Alias('test-%s' % progName, run_program)
        env['testNames'].append(progName)
    else:
        test_results.failed['test-%s (googletest disabled)' % progName] = 1

    if os.path.exists(passedFile.abspath):
        Alias('test-reset', testenv.Command('reset-%s%s' % (subdir, progName),
                                            [], [Delete(passedFile.abspath)]))


def addPythonTest(testname, subdir, script, interpreter, outfile,
                  args='', dependencies=(), env_vars={}, optional=False):
    """
    Create targets for running and resetting a test script.
    """
    def scriptRunner(target, source, env):
        unittest_outfile = File("#test/work/python-results.txt").abspath
        pytest_outfile = File("#test/work/pytest.xml").abspath
        """Scons Action to run a test script using the specified interpreter"""
        workDir = Dir('#test/work').abspath
        passedFile = target[0]
        test_results.tests.pop(passedFile.name, None)
        if not os.path.isdir(workDir):
            os.mkdir(workDir)
        if os.path.exists(unittest_outfile):
            os.remove(unittest_outfile)
        if os.path.exists(pytest_outfile):
            os.remove(pytest_outfile)

        environ = dict(env['ENV'])
        for k,v in env_vars.items():
            print(k,v)
            environ[k] = v

        cmdargs = args.split()
        if env["fast_fail_tests"]:
            cmdargs.insert(0, "fast_fail")
        if env["skip_slow_tests"]:
            environ["CT_SKIP_SLOW"] = "1"

        code = subprocess.call(
            [env.subst(interpreter), source[0].abspath] + cmdargs,
            env=environ
        )
        if not code:
            # Test was successful
            with open(target[0].path, 'w') as passed_file:
                passed_file.write(time.asctime()+'\n')
        elif env["fast_fail_tests"]:
            sys.exit(1)

        failures = 0
        if os.path.exists(unittest_outfile):
            # Determine individual test status
            for line in open(outfile):
                status, name = line.strip().split(': ', 1)
                if status == 'PASS':
                    test_results.passed[':'.join((testname,name))] = 1
                elif status in ('FAIL', 'ERROR'):
                    test_results.failed[':'.join((testname,name))] = 1
                    failures += 1
        elif os.path.exists(pytest_outfile):
            results = ElementTree.parse(pytest_outfile)
            for test in results.findall('.//testcase'):
                class_name = test.get('classname')
                if class_name.startswith("build.python.cantera.test."):
                    class_name = class_name[26:]
                test_name = "python: {}.{}".format(class_name, test.get('name'))
                if test.findall('failure'):
                    test_results.failed[test_name] = 1
                    failures += 1
                else:
                    test_results.passed[test_name] = 1

        if code and failures == 0:
            # Failure, but unable to determine status of individual tests. This
            # gets counted as many failures.
            test_results.failed[testname +
                ' ***no results for entire test suite***'] = 100

    testenv = localenv.Clone()
    passedFile = File(pjoin(subdir, '%s.passed' % testname))
    PASSED_FILES[testname] = str(passedFile)

    run_program = testenv.Command(passedFile, pjoin('#test', subdir, script), scriptRunner)
    for dep in dependencies:
        if isinstance(dep, str):
            dep = File(pjoin(subdir, dep))
        testenv.Depends(run_program, dep)
    if not optional:
        testenv.Depends(env['test_results'], run_program)
        test_results.tests[passedFile.name] = True
    if os.path.exists(passedFile.abspath):
        Alias('test-reset', testenv.Command('reset-%s%s' % (subdir, testname),
                                            [], [Delete(passedFile.abspath)]))

    return run_program


def addMatlabTest(script, testName, dependencies=None, env_vars=()):
    def matlabRunner(target, source, env):
        passedFile = target[0]
        del test_results.tests[passedFile.name]
        workDir = Dir('#test/work').abspath
        if not os.path.isdir(workDir):
            os.mkdir(workDir)
        outfile = pjoin(workDir, 'matlab-results.txt')
        runCommand = "%s('%s'); exit" % (source[0].name[:-2], outfile)
        if os.name == 'nt':
            matlabOptions = ['-nojvm','-nosplash','-wait']
        else:
            matlabOptions = ['-nojvm','-nodisplay']
        if os.path.exists(outfile):
            os.remove(outfile)

        environ = dict(os.environ)
        environ.update(env['ENV'])
        environ.update(env_vars)
        code = subprocess.call([pjoin(env['matlab_path'], 'bin', 'matlab')] +
                               matlabOptions + ['-r', runCommand],
                               env=environ, cwd=Dir('#test/matlab').abspath)
        if code and env["fast_fail_tests"]:
            sys.exit(1)
        try:
            with open(outfile, "r") as output_file:
                results = output_file.read()
        except EnvironmentError: # TODO: replace with 'FileNotFoundError' after end of Python 2.7 support
            test_results.failed['Matlab' +
                ' ***no results for entire test suite***'] = 100
            return

        print('-------- Matlab test results --------')
        print(results)
        print('------ end Matlab test results ------')

        passed = True
        for line in results.split('\n'):
            line = line.strip()
            if not line:
                continue
            label = line.split()[0]
            if 'seconds' not in line: # Headers for test suite sections
                section = line
            elif 'test' in line and 'matlab' in line: # skip overall summary line
                continue
            elif label == section: # skip summary line for each section
                continue
            elif 'FAILED' in line:
                test_results.failed['.'.join(['Matlab', section, label])] = 1
                passed = False
            elif 'passed' in line:
                test_results.passed['.'.join(['Matlab', section, label])] = 1

        if passed:
            open(target[0].path, 'w').write(time.asctime()+'\n')

    testenv = localenv.Clone()
    passedFile = File('matlab/%s.passed' % (script))
    PASSED_FILES[testName] = str(passedFile)
    test_results.tests[passedFile.name] = True
    run_program = testenv.Command(passedFile, pjoin('matlab', script), matlabRunner)

    dependencies = (dependencies or []) + localenv['matlab_extension']
    for dep in dependencies:
        if isinstance(dep, str):
            dep = File(pjoin('matlab', dep))
        testenv.Depends(run_program, dep)

    testenv.Depends(testenv['test_results'], run_program)
    if os.path.exists(passedFile.abspath):
        Alias('test-reset', testenv.Command('reset-%s%s' % ('matlab', script),
                                            [], [Delete(passedFile.abspath)]))

    return run_program

# Instantiate tests
addTestProgram('general', 'general')
addTestProgram('thermo', 'thermo')
addTestProgram('equil', 'equil')
addTestProgram('kinetics', 'kinetics')
addTestProgram('transport', 'transport')

python_subtests = ['']
test_root = '#interfaces/cython/cantera/test'
for f in multi_glob(localenv, test_root, '^test_*.py'):
    python_subtests.append(f.name[5:-3])

if localenv['python_package'] == 'full':
    # Create test aliases for individual test modules (e.g. test-python-thermo;
    # not run as part of the main suite) and a single test runner with all the
    # tests (test-python) for the main suite.
    for subset in python_subtests:
        name = 'python-' + subset if subset else 'python'
        pyTest = addPythonTest(
            name, 'python', 'runCythonTests.py',
            args=subset,
            interpreter='$python_cmd',
            outfile=File('#test/work/python-results.txt').abspath,
            dependencies=(localenv['python_module'] + localenv['python_extension'] +
                          multi_glob(localenv, test_root, 'py')),
            optional=bool(subset))
        localenv.Alias('test-' + name, pyTest)
        env['testNames'].append(name)

if localenv['matlab_toolbox'] == 'y':
    matlabTest = addMatlabTest('runCanteraTests.m', 'matlab',
                               dependencies=multi_glob(localenv, 'matlab', 'm'))
    localenv.Alias('test-matlab', matlabTest)
    env['testNames'].append('matlab')

# Force explicitly-named tests to run even if SCons thinks they're up to date
for command in COMMAND_LINE_TARGETS:
    if command.startswith('test-'):
        name = command[5:]
        if name in PASSED_FILES and os.path.exists(PASSED_FILES[name]):
            os.remove(PASSED_FILES[name])