memtest86plus/tests/vec_prsg.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2004-2026 Sam Demeulemeester.
//
// Vector-wide pseudo-random sequence primitives shared by the memory tests.
//
// Memory is filled and checked with per-lane xorshift streams, one whole
// vector block at a time. On x86_64 the work is dispatched at runtime to
// AVX2 or SSE2 kernels with non-temporal stores (tests/x86/vec_prsg_*.c);
// other builds use the scalar kernels below, which implement the same
// algorithm. The check functions regenerate the sequence instead of storing
// it: xorshift is an invertible linear map, so the descending check simply
// steps the lane states backwards.
//
// SIMD dispatch approach inspired by earlier experimental work by
// Lionel Debroux.

#include <stdbool.h>
#include <stdint.h>

#include "simd.h"

#include "error.h"
#include "test.h"

#include "test_helper.h"

#include "vec_prsg.h"

//------------------------------------------------------------------------------
// Constants
//------------------------------------------------------------------------------

// Arbitrary distinct constants used to decorrelate the per-lane xorshift
// streams seeded from a single per-round seed.
#if TESTWORD_WIDTH > 32
static const testword_t lane_salt[VEC_LANES] = {
    UINT64_C(0x9e3779b97f4a7c15),
    UINT64_C(0xbf58476d1ce4e5b9),
    UINT64_C(0x94d049bb133111eb),
    UINT64_C(0x2545f4914f6cdd1d)
};
#else
static const testword_t lane_salt[VEC_LANES] = {
    0x9e3779b9,
    0xbf58476d,
    0x94d049bb,
    0x2545f491
};
#endif

//------------------------------------------------------------------------------
// Private Functions
//------------------------------------------------------------------------------

// Inverts one y = x ^ (x << shift) step of the xorshift generator by repeated
// squaring, doubling the shift until it exceeds the word width.
static inline testword_t prsg_undo_lshift(testword_t y, int shift)
{
    for (int k = shift; k < TESTWORD_WIDTH; k *= 2) {
        y ^= y << k;
    }
    return y;
}

// Inverts one y = x ^ (x >> shift) step of the xorshift generator by repeated
// squaring, doubling the shift until it exceeds the word width.
static inline testword_t prsg_undo_rshift(testword_t y, int shift)
{
    for (int k = shift; k < TESTWORD_WIDTH; k *= 2) {
        y ^= y >> k;
    }
    return y;
}

// Returns the previous word in the pseudo-random sequence generated by prsg().
static inline testword_t prsg_inverse(testword_t state)
{
#if TESTWORD_WIDTH > 32
    state = prsg_undo_lshift(state, 17);
    state = prsg_undo_rshift(state, 7);
    state = prsg_undo_lshift(state, 13);
#else
    state = prsg_undo_lshift(state, 5);
    state = prsg_undo_rshift(state, 17);
    state = prsg_undo_lshift(state, 13);
#endif
    return state;
}

//------------------------------------------------------------------------------
// Scalar kernels (all architectures)
//
// These process nblocks whole vector blocks starting at p (the descending
// check processes them from the top down). The lane states are carried in
// memory between invocations, so no SIMD state is live across calls to
// data_error(), do_tick(), or the thread barriers.
//------------------------------------------------------------------------------

static void vec_fill_scalar(vec_state_t *st, testword_t *p, size_t nblocks, bool splat)
{
    for (size_t k = 0; k < nblocks; k++) {
        for (int l = 0; l < VEC_LANES; l++) {
            write_word(&p[l], st->lane[l]);
        }
        p += VEC_LANES;
        if (!splat) {
            for (int l = 0; l < VEC_LANES; l++) {
                st->lane[l] = prsg(st->lane[l]);
            }
        }
    }
}

static void vec_check_fwd_scalar(vec_state_t *st, testword_t *p, size_t nblocks, bool splat, bool use_for_badram)
{
    for (size_t k = 0; k < nblocks; k++) {
        for (int l = 0; l < VEC_LANES; l++) {
            testword_t expect = st->lane[l];
            testword_t actual = read_word(&p[l]);
            if (unlikely(actual != expect)) {
                data_error(&p[l], expect, actual, use_for_badram);
            }
            write_word(&p[l], ~expect);
        }
        p += VEC_LANES;
        if (!splat) {
            for (int l = 0; l < VEC_LANES; l++) {
                st->lane[l] = prsg(st->lane[l]);
            }
        }
    }
}

static void vec_check_rev_scalar(vec_state_t *st, testword_t *p, size_t nblocks, bool splat, bool use_for_badram)
{
    for (size_t k = nblocks; k-- > 0; ) {
        testword_t *q = p + k * VEC_LANES;
        if (!splat) {
            for (int l = 0; l < VEC_LANES; l++) {
                st->lane[l] = prsg_inverse(st->lane[l]);
            }
        }
        for (int l = 0; l < VEC_LANES; l++) {
            testword_t expect = ~st->lane[l];
            testword_t actual = read_word(&q[l]);
            if (unlikely(actual != expect)) {
                data_error(&q[l], expect, actual, use_for_badram);
            }
            write_word(&q[l], st->lane[l]);
        }
    }
}

//------------------------------------------------------------------------------
// Public Functions
//------------------------------------------------------------------------------

void seed_lanes(vec_state_t *st, testword_t seed, bool splat)
{
    if (splat) {
        testword_t s = seed != 0 ? seed : lane_salt[0];
        s = prsg(s);
        for (int l = 0; l < VEC_LANES; l++) {
            st->lane[l] = s;
        }
    } else {
        for (int l = 0; l < VEC_LANES; l++) {
            testword_t s = seed ^ lane_salt[l];
            if (s == 0) {
                s = lane_salt[l];
            }
            st->lane[l] = prsg(s);
        }
    }
}

void vec_fill(vec_state_t *st, testword_t *p, size_t nblocks, bool splat)
{
#if defined(__x86_64__)
    if (simd_tier == SIMD_AVX2) {
        vec_fill_avx2(st, p, nblocks, splat);
        return;
    }
    if (simd_tier == SIMD_SSE2) {
        vec_fill_sse2(st, p, nblocks, splat);
        return;
    }
#elif defined(__aarch64__)
    if (simd_tier == SIMD_NEON) {
        vec_fill_neon(st, p, nblocks, splat);
        return;
    }
#elif defined(__loongarch_lp64)
    if (simd_tier == SIMD_LASX) {
        vec_fill_lasx(st, p, nblocks, splat);
        return;
    }
#endif
    vec_fill_scalar(st, p, nblocks, splat);
}

void vec_check_fwd(vec_state_t *st, testword_t *p, size_t nblocks, bool splat, bool use_for_badram)
{
#if defined(__x86_64__)
    if (simd_tier != SIMD_NONE) {
        while (nblocks > 0) {
            size_t done = (simd_tier == SIMD_AVX2) ? vec_scan_fwd_avx2(st, p, nblocks, splat)
                                                   : vec_scan_fwd_sse2(st, p, nblocks, splat);
            p += done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block. Report and fix it up,
            // then resume the scan on the remaining blocks.
            vec_check_fwd_scalar(st, p, 1, splat, use_for_badram);
            p += VEC_LANES;
            nblocks--;
        }
        return;
    }
#elif defined(__aarch64__)
    if (simd_tier == SIMD_NEON) {
        while (nblocks > 0) {
            size_t done = vec_scan_fwd_neon(st, p, nblocks, splat);
            p += done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block. Report and fix it up,
            // then resume the scan on the remaining blocks.
            vec_check_fwd_scalar(st, p, 1, splat, use_for_badram);
            p += VEC_LANES;
            nblocks--;
        }
        return;
    }
#elif defined(__loongarch_lp64)
    if (simd_tier == SIMD_LASX) {
        while (nblocks > 0) {
            size_t done = vec_scan_fwd_lasx(st, p, nblocks, splat);
            p += done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block. Report and fix it up,
            // then resume the scan on the remaining blocks.
            vec_check_fwd_scalar(st, p, 1, splat, use_for_badram);
            p += VEC_LANES;
            nblocks--;
        }
        return;
    }
#endif
    vec_check_fwd_scalar(st, p, nblocks, splat, use_for_badram);
}

void vec_check_rev(vec_state_t *st, testword_t *p, size_t nblocks, bool splat, bool use_for_badram)
{
#if defined(__x86_64__)
    if (simd_tier != SIMD_NONE) {
        testword_t *q = p + (nblocks - 1) * VEC_LANES;
        while (nblocks > 0) {
            size_t done = (simd_tier == SIMD_AVX2) ? vec_scan_rev_avx2(st, q, nblocks, splat)
                                                   : vec_scan_rev_sse2(st, q, nblocks, splat);
            q -= done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block, with the lane states
            // already stepped back to it. Report and fix it up, then resume.
            // Unlike the forward case, the scalar kernel cannot be reused
            // here as it would step the lane states back a second time.
            for (int l = 0; l < VEC_LANES; l++) {
                testword_t expect = ~st->lane[l];
                testword_t actual = read_word(&q[l]);
                if (unlikely(actual != expect)) {
                    data_error(&q[l], expect, actual, use_for_badram);
                }
                write_word(&q[l], st->lane[l]);
            }
            q -= VEC_LANES;
            nblocks--;
        }
        return;
    }
#elif defined(__aarch64__)
    if (simd_tier == SIMD_NEON) {
        testword_t *q = p + (nblocks - 1) * VEC_LANES;
        while (nblocks > 0) {
            size_t done = vec_scan_rev_neon(st, q, nblocks, splat);
            q -= done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block.
            for (int l = 0; l < VEC_LANES; l++) {
                testword_t expect = ~st->lane[l];
                testword_t actual = read_word(&q[l]);
                if (unlikely(actual != expect)) {
                    data_error(&q[l], expect, actual, use_for_badram);
                }
                write_word(&q[l], st->lane[l]);
            }
            q -= VEC_LANES;
            nblocks--;
        }
        return;
    }
#elif defined(__loongarch_lp64)
    if (simd_tier == SIMD_LASX) {
        testword_t *q = p + (nblocks - 1) * VEC_LANES;
        while (nblocks > 0) {
            size_t done = vec_scan_rev_lasx(st, q, nblocks, splat);
            q -= done * VEC_LANES;
            nblocks -= done;
            if (nblocks == 0) {
                break;
            }
            // The scan stopped at a mismatching block.
            for (int l = 0; l < VEC_LANES; l++) {
                testword_t expect = ~st->lane[l];
                testword_t actual = read_word(&q[l]);
                if (unlikely(actual != expect)) {
                    data_error(&q[l], expect, actual, use_for_badram);
                }
                write_word(&q[l], st->lane[l]);
            }
            q -= VEC_LANES;
            nblocks--;
        }
        return;
    }
#endif
    vec_check_rev_scalar(st, p, nblocks, splat, use_for_badram);
}