Rework memory mapping to allow for larger program size (#54)

* Improve abstraction in vmem.h and limit memory benchmarking to first 2GB.

The third GB may get used for remapping memory regions that are only
accessed during startup, so it's not safe to use it for the memory
speed tests.

* Fix calculation of end limit for locating memory benchmark workspace.

* Document vmem.h.

* Use window number, not current start address, to detect first window.

* Increase the program low-load range from 1MB to 4MB and make more robust.

If the BIOS has reserved some parts of low memory, there may not be
enough contiguous space left to load the program there (issue #49).
So increase the low-load range to include the first 3MB of high
memory. Also guard against the program being initially loaded
straddling the new boundary.

Co-authored-by: Martin Whitaker <memtest@martin-whitaker.me.uk>

app/main.c

@@ -58,6 +58,10 @@
 #define TEST_INTERRUPT      0
 #endif
 
+#define LOW_LOAD_LIMIT      SIZE_C(4,MB)  // must be a multiple of the page size
+
+#define HIGH_LOAD_LIMIT     (VM_PINNED_SIZE << PAGE_SHIFT)
+
 //------------------------------------------------------------------------------
 // Private Variables
 //------------------------------------------------------------------------------
@@ -79,7 +83,6 @@ static bool             rerun_test = false;
 
 static bool             dummy_run  = false;
 
-static int              window_num   = 0;
 static uintptr_t        window_start = 0;
 static uintptr_t        window_end   = 0;
 
@@ -110,6 +113,8 @@ int         vm_map_size = 0;
 int         pass_num = 0;
 int         test_num = 0;
 
+int         window_num = 0;
+
 bool        restart = false;
 bool        bail    = false;
 
@@ -145,7 +150,7 @@ static void run_at(uintptr_t addr, int my_cpu)
 
     if (my_cpu == 0) {
         // Copy the program code and all data except the stacks.
-        memcpy((void *)addr, (void *)_start, _stacks - _start);
+        memmove((void *)addr, (void *)_start, _stacks - _start);
         // Copy the thread-local storage.
         size_t locals_offset = _stacks - _start + BSP_STACK_SIZE - LOCALS_SIZE;
         for (int cpu_num = 0; cpu_num < num_available_cpus; cpu_num++) {
@@ -166,7 +171,7 @@ static void run_at(uintptr_t addr, int my_cpu)
 static bool set_load_addr(uintptr_t *load_addr, size_t program_size, uintptr_t lower_limit, uintptr_t upper_limit)
 {
     uintptr_t current_start = (uintptr_t)_start;
-    if (current_start >= lower_limit && current_start < upper_limit) {
+    if (current_start >= lower_limit && (current_start + program_size) <= upper_limit) {
         *load_addr = current_start;
         return true;
     }
@@ -174,7 +179,7 @@ static bool set_load_addr(uintptr_t *load_addr, size_t program_size, uintptr_t l
     for (int i = 0; i < pm_map_size; i++) {
         uintptr_t try_start = pm_map[i].start << PAGE_SHIFT;
         uintptr_t try_limit = pm_map[i].end   << PAGE_SHIFT;
-        if (try_start == 0) try_start = 0x1000;
+        if (try_start < lower_limit) try_start = lower_limit;
         uintptr_t try_end   = try_start + program_size;
         if (try_end > try_limit) continue;
 
@@ -259,8 +264,8 @@ static void global_init(void)
 
     size_t program_size = (_stacks - _start) + BSP_STACK_SIZE + (num_enabled_cpus - 1) * AP_STACK_SIZE;
 
-    bool load_addr_ok = set_load_addr(& low_load_addr, program_size,            0, SIZE_C(1,MB))
-                     && set_load_addr(&high_load_addr, program_size, SIZE_C(1,MB), SIZE_C(2,GB));
+    bool load_addr_ok = set_load_addr(& low_load_addr, program_size,         0x1000,  LOW_LOAD_LIMIT)
+                     && set_load_addr(&high_load_addr, program_size, LOW_LOAD_LIMIT, HIGH_LOAD_LIMIT);
 
     trace(0, "program size %ikB", (int)(program_size / 1024));
     trace(0, " low_load_addr %0*x", 2*sizeof(uintptr_t),  low_load_addr);
@@ -372,7 +377,7 @@ static void test_all_windows(int my_cpu)
                 // Relocation may disrupt the test.
                 window_num = 1;
             }
-            if (window_num == 0 && pm_limit_lower >= high_load_addr) {
+            if (window_num == 0 && pm_limit_lower >= LOW_LOAD_LIMIT) {
                 // Avoid unnecessary relocation.
                 window_num = 1;
             }
@@ -395,10 +400,10 @@ static void test_all_windows(int my_cpu)
             switch (window_num) {
               case 0:
                 window_start = 0;
-                window_end   = (high_load_addr >> PAGE_SHIFT);
+                window_end   = (LOW_LOAD_LIMIT >> PAGE_SHIFT);
                 break;
               case 1:
-                window_start = (high_load_addr >> PAGE_SHIFT);
+                window_start = (LOW_LOAD_LIMIT >> PAGE_SHIFT);
                 window_end   = VM_WINDOW_SIZE;
                 break;
               default:

app/test.h

@@ -92,6 +92,10 @@ extern int pass_num;
  * The current test number.
  */
 extern int test_num;
+/**
+ * The current window number.
+ */
+extern int window_num;
 
 /**
  * A flag indicating that testing should be restarted due to a configuration

system/cpuinfo.c

@@ -1021,8 +1021,8 @@ static void measure_memory_bandwidth(void)
     }
 
     // Locate enough free space for tests. We require the space to be mapped into
-    // our virtual address space, which limits us to the first 3GB.
-    for (int i = 0; i < pm_map_size && pm_map[i].start < VM_BENCH_WINDOW_SIZE; i++) {
+    // our virtual address space, which limits us to the first 2GB.
+    for (int i = 0; i < pm_map_size && pm_map[i].start < VM_PINNED_SIZE; i++) {
         uintptr_t try_start = pm_map[i].start << PAGE_SHIFT;
         uintptr_t try_end   = try_start + mem_test_len * 2;
 
@@ -1042,7 +1042,7 @@ static void measure_memory_bandwidth(void)
             try_end   = try_start + mem_test_len * 2;
         }
 
-        uintptr_t end_limit = pm_map[i].end < VM_BENCH_WINDOW_SIZE ? pm_map[i].end << PAGE_SHIFT : VM_BENCH_WINDOW_SIZE;
+        uintptr_t end_limit = (pm_map[i].end < VM_PINNED_SIZE ? pm_map[i].end : VM_PINNED_SIZE) << PAGE_SHIFT;
         if (try_end <= end_limit) {
             bench_start_adr = try_start;
             break;

system/vmem.h

@@ -7,6 +7,11 @@
  * Provides functions to handle physical memory page mapping into virtual
  * memory.
  *
+ * The startup code sets up the paging tables to give us a 4GB virtual address
+ * space, initially identity mapped to the first 4GB of physical memory. We
+ * leave the lower 2GB permanently mapped, and use the upper 2GB for mapping
+ * the remaining physical memory as required.
+ *
  *//*
  * Copyright (C) 2020-2022 Martin Whitaker.
  */
@@ -17,17 +22,82 @@
 
 #include "memsize.h"
 
-#define VM_WINDOW_SIZE           PAGE_C(1,GB)
-#define VM_BENCH_WINDOW_SIZE     PAGE_C(3,GB)
+/**
+ * The size of the physical memory region (in pages) that is permanently mapped
+ * into virtual memory, starting at virtual address 0.
+ */
+#define VM_PINNED_SIZE  PAGE_C(2,GB)
 
+/**
+ * The size of a physical memory region (in pages) that can be mapped into
+ * virtual memory by a call to map_window().
+ */
+#define VM_WINDOW_SIZE  PAGE_C(1,GB)
+
+/**
+ * Maps a physical memory region into the upper 2GB of virtual memory. The
+ * virtual address will have the same alignment within a page as the physical
+ * address.
+ *
+ * \param base_addr         - the physical byte address of the region.
+ * \param size              - the region size in bytes.
+ * \param only_for_startup  - if true, the region will remain mapped until the
+ *                            first call to map_window(), otherwise it will be
+ *                            permanently mapped.
+ *
+ * \returns
+ * On success, the mapped address in virtual memory, On failure, 0.
+ */
 uintptr_t map_region(uintptr_t base_addr, size_t size, bool only_for_startup);
 
+/**
+ * Maps a \ref VM_WINDOW_SIZE region of physical memory into the upper 2GB of
+ * virtual memory. The physical memory region must be aligned on a \ref
+ * VM_WINDOW_SIZE boundary. The virtual address will be similarly aligned.
+ * The region will remain mapped until the next call to map_window().
+ *
+ * \param start_page        - the physical page number of the region.
+ *
+ * \returns
+ * On success, true. On failure, false.
+ */
 bool map_window(uintptr_t start_page);
 
+/**
+ * Returns a virtual memory pointer to the first word of the specified physical
+ * memory page. Physical memory pages above \ref VM_PINNED_SIZE must have been
+ * mapped by a call to map_window() prior to calling this function.
+ *
+ * \param page              - the physical page number.
+ *
+ * \returns
+ * A pointer to the first word of the page.
+ */
 void *first_word_mapping(uintptr_t page);
 
+/**
+ * Returns a virtual memory pointer to the last word of the specified physical
+ * memory page. Physical memory pages above \ref VM_PINNED_SIZE must have been
+ * mapped by a call to map_window() prior to calling this function.
+ *
+ * \param page              - the physical page number.
+ * \param word_size         - the size of a word in bytes.
+ *
+ * \returns
+ * A pointer to the last word of the page.
+ */
 void *last_word_mapping(uintptr_t page, size_t word_size);
 
+/**
+ * Returns the page number of the physical memory page containing the specified
+ * virtual memory address. The specified address must either be permanently
+ * mapped or mapped by a call to map_window() prior to calling this function.
+ *
+ * \param addr              - the virtual memory address.
+ *
+ * \returns
+ * The corresponding physical page number.
+ */
 uintptr_t page_of(void *addr);
 
 #endif // VMEM_H

tests/tests.c

@@ -89,8 +89,8 @@ int ticks_per_test[NUM_PASS_TYPES][NUM_TEST_PATTERNS];
 int run_test(int my_cpu, int test, int stage, int iterations)
 {
     if (my_cpu == master_cpu) {
-        if ((uintptr_t)&_start > SIZE_C(1,MB)) {
-            // Relocated so we need to test all selected lower memory.
+        if (window_num == 0) {
+            // First window, so we need to test all selected lower memory.
             vm_map[0].start = first_word_mapping(pm_limit_lower);
 
             // For USB_WORKAROUND.