RTS Refactoring (#4204)

* Make `memory_check` feature flag compilable again.
* Eliminate compiler warnings in RTS test.
* Support `region0` in RTS tests. (Btw: The `region0` root has only been moved because it is not an array, not omitted. Previously, the declared `region0` parameters were unused.)

rts/motoko-rts-tests/src/gc.rs

@@ -76,22 +76,19 @@ fn test_heaps() -> Vec<TestHeap> {
             ],
             roots: vec![0, 2, 3],
             continuation_table: vec![0],
-            region0_ptr_loc: vec![0],
         },
         // Tests pointing to the same object in multiple fields of an object. Also has unreachable
         // objects.
         TestHeap {
             heap: vec![(0, vec![]), (1, vec![]), (2, vec![])],
             roots: vec![1],
             continuation_table: vec![0, 0],
-            region0_ptr_loc: vec![0],
         },
         // Root points backwards in heap. Caught a bug in mark-compact collector.
         TestHeap {
             heap: vec![(0, vec![]), (1, vec![2]), (2, vec![1])],
             roots: vec![2],
             continuation_table: vec![],
-            region0_ptr_loc: vec![0],
         },
     ]
 }
@@ -109,7 +106,6 @@ struct TestHeap {
     heap: Vec<(ObjectIdx, Vec<ObjectIdx>)>,
     roots: Vec<ObjectIdx>,
     continuation_table: Vec<ObjectIdx>,
-    region0_ptr_loc: Vec<ObjectIdx>,
 }
 
 /// Test all GC implementations with the given heap
@@ -120,7 +116,6 @@ fn test_gcs(heap_descr: &TestHeap) {
             &heap_descr.heap,
             &heap_descr.roots,
             &heap_descr.continuation_table,
-            &heap_descr.region0_ptr_loc,
         );
     }
 
@@ -132,9 +127,8 @@ fn test_gc(
     refs: &[(ObjectIdx, Vec<ObjectIdx>)],
     roots: &[ObjectIdx],
     continuation_table: &[ObjectIdx],
-    region0_ptr_loc: &[ObjectIdx],
 ) {
-    let mut heap = MotokoHeap::new(refs, roots, continuation_table, region0_ptr_loc, gc);
+    let mut heap = MotokoHeap::new(refs, roots, continuation_table, gc);
 
     initialize_gc(&mut heap);
 
@@ -148,6 +142,7 @@ fn test_gc(
         heap.heap_base_offset(),
         heap.heap_ptr_offset(),
         heap.continuation_table_ptr_offset(),
+        heap.region0_ptr_offset(),
     );
 
     for round in 0..3 {
@@ -156,6 +151,7 @@ fn test_gc(
         let heap_base_offset = heap.heap_base_offset();
         let heap_ptr_offset = heap.heap_ptr_offset();
         let continuation_table_ptr_offset = heap.continuation_table_ptr_offset();
+        let region0_ptr_offset = heap.region0_ptr_offset();
         check_dynamic_heap(
             check_all_reclaimed, // check for unreachable objects
             refs,
@@ -165,6 +161,7 @@ fn test_gc(
             heap_base_offset,
             heap_ptr_offset,
             continuation_table_ptr_offset,
+            region0_ptr_offset,
         );
     }
 }
@@ -220,6 +217,7 @@ fn check_dynamic_heap(
     heap_base_offset: usize,
     heap_ptr_offset: usize,
     continuation_table_ptr_offset: usize,
+    region0_ptr_offset: usize,
 ) {
     let incremental = cfg!(feature = "incremental_gc");
     let objects_map: FxHashMap<ObjectIdx, &[ObjectIdx]> = objects
@@ -236,6 +234,8 @@ fn check_dynamic_heap(
     let continuation_table_addr = unskew_pointer(read_word(heap, continuation_table_ptr_offset));
     let continuation_table_offset = continuation_table_addr as usize - heap.as_ptr() as usize;
 
+    let region0_addr = unskew_pointer(read_word(heap, region0_ptr_offset));
+
     while offset < heap_ptr_offset {
         let object_offset = offset;
 
@@ -276,6 +276,13 @@ fn check_dynamic_heap(
                 // in-heap mark stack blobs
                 let length = read_word(heap, offset);
                 offset += WORD_SIZE + length as usize;
+            } else if tag == TAG_REGION {
+                if !is_forwarded {
+                    assert_eq!(address, region0_addr as usize);
+                }
+                offset += (size_of::<Region>() - size_of::<Obj>())
+                    .to_bytes()
+                    .as_usize();
             } else {
                 if incremental {
                     assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
@@ -502,10 +509,12 @@ impl GC {
             }
 
             GC::Generational => {
-                use motoko_rts::gc::generational::{
+                use motoko_rts::gc::{
+                    generational::{
+                        write_barrier::{LAST_HP, REMEMBERED_SET},
+                        GenerationalGC, Strategy,
+                    },
                     remembered_set::RememberedSet,
-                    write_barrier::{LAST_HP, REMEMBERED_SET},
-                    GenerationalGC, Strategy,
                 };
 
                 let strategy = match _round {
@@ -546,7 +555,7 @@ impl GC {
     fn run(&self, heap: &mut MotokoHeap, _round: usize) -> bool {
         let static_roots = Value::from_ptr(heap.static_root_array_address());
         let continuation_table_ptr_address = heap.continuation_table_ptr_address() as *mut Value;
-        let region0_ptr_location = heap.region0_ptr_location() as *mut Value;
+        let region0_ptr_address = heap.region0_ptr_address() as *mut Value;
 
         match self {
             GC::Incremental => unsafe {
@@ -556,7 +565,7 @@ impl GC {
                     let roots = motoko_rts::gc::incremental::roots::Roots {
                         static_roots,
                         continuation_table_location: continuation_table_ptr_address,
-                        region0_ptr_location,
+                        region0_ptr_location: region0_ptr_address,
                     };
                     IncrementalGC::instance(heap, incremental_gc_state())
                         .empty_call_stack_increment(roots);

rts/motoko-rts-tests/src/gc/generational/remembered_set.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 
 use crate::memory::TestMemory;
-use motoko_rts::gc::generational::remembered_set::{
+use motoko_rts::gc::remembered_set::{
     RememberedSet, INITIAL_TABLE_LENGTH, OCCUPATION_THRESHOLD_PERCENT,
 };
 use motoko_rts::types::{Value, Words};

rts/motoko-rts-tests/src/gc/heap.rs

@@ -38,7 +38,6 @@ impl MotokoHeap {
         map: &[(ObjectIdx, Vec<ObjectIdx>)],
         roots: &[ObjectIdx],
         continuation_table: &[ObjectIdx],
-        region0_ptr_loc: &[ObjectIdx],
         gc: GC,
     ) -> MotokoHeap {
         MotokoHeap {
@@ -106,8 +105,14 @@ impl MotokoHeap {
         self.inner.borrow().continuation_table_ptr_address()
     }
 
-    /// Get the address of the continuation table pointer
-    pub fn region0_ptr_location(&self) -> usize {
+    /// Get the offset of the region0 pointer
+    pub fn region0_ptr_offset(&self) -> usize {
+        self.inner.borrow().region0_ptr_location_offset
+    }
+
+    /// Get the address of the region0 pointer
+    #[incremental_gc]
+    pub fn region0_ptr_address(&self) -> usize {
         self.inner.borrow().region0_ptr_address()
     }
 
@@ -149,10 +154,14 @@ struct MotokoHeapInner {
 
     /// Offset of the continuation table pointer.
     ///
-    /// Reminder: this location is in static heap and will have pointer to an array in dynamic
-    /// heap.
+    /// Reminder: this location is in static heap and will have a pointer to an array in
+    /// the dynamic heap.
     continuation_table_ptr_offset: usize,
 
+    /// Offset of the region0 pointer.
+    ///
+    /// Reminder: this location is in static heap and will have a pointer to a region in
+    /// the dynamic heap.
     region0_ptr_location_offset: usize,
 }
 
@@ -202,6 +211,8 @@ impl MotokoHeapInner {
         self.offset_to_address(self.continuation_table_ptr_offset)
     }
 
+    /// Get the address of the region0 pointer
+    #[incremental_gc]
     fn region0_ptr_address(&self) -> usize {
         self.offset_to_address(self.region0_ptr_location_offset)
     }
@@ -231,16 +242,20 @@ impl MotokoHeapInner {
             + 2)
             * WORD_SIZE;
 
-        let dynamic_heap_size_without_continuation_table_bytes = {
+        let dynamic_heap_size_without_roots = {
             let object_headers_words = map.len() * (size_of::<Array>().as_usize() + 1);
             let references_words = map.iter().map(|(_, refs)| refs.len()).sum::<usize>();
             (object_headers_words + references_words) * WORD_SIZE
         };
 
-        let dynamic_heap_size_bytes = dynamic_heap_size_without_continuation_table_bytes
-            + (size_of::<Array>() + Words(continuation_table.len() as u32))
-                .to_bytes()
-                .as_usize();
+        let continuation_table_size = (size_of::<Array>() + Words(continuation_table.len() as u32))
+            .to_bytes()
+            .as_usize();
+
+        let region0_size = size_of::<Region>().to_bytes().as_usize();
+
+        let dynamic_heap_size_bytes =
+            dynamic_heap_size_without_roots + continuation_table_size + region0_size;
 
         let total_heap_size_bytes = static_heap_size_bytes + dynamic_heap_size_bytes;
 
@@ -259,23 +274,26 @@ impl MotokoHeapInner {
         let realign = (32 - (heap.as_ptr() as usize + static_heap_size_bytes) % 32) % 32;
         assert_eq!(realign % 4, 0);
 
-        // Maps `ObjectIdx`s into their offsets in the heap
+        // Maps `ObjectIdx`s into their offsets in the heap.
         let object_addrs: FxHashMap<ObjectIdx, usize> = create_dynamic_heap(
             map,
             continuation_table,
             &mut heap[static_heap_size_bytes + realign..heap_size + realign],
         );
 
-        // Closure table pointer is the last word in static heap
+        // Closure table pointer is the second last word in the static heap.
         let continuation_table_ptr_offset = static_heap_size_bytes - WORD_SIZE * 2;
 
+        // Region0 pointer is the very last word in the static heap.
         let region0_ptr_location_offset = static_heap_size_bytes - WORD_SIZE;
 
         create_static_heap(
             roots,
             &object_addrs,
             continuation_table_ptr_offset,
-            static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes,
+            static_heap_size_bytes + dynamic_heap_size_without_roots,
+            region0_ptr_location_offset,
+            static_heap_size_bytes + dynamic_heap_size_without_roots + continuation_table_size,
             &mut heap[realign..static_heap_size_bytes + realign],
         );
 
@@ -472,6 +490,8 @@ fn create_dynamic_heap(
         .to_bytes()
         .as_usize()
         + n_fields * WORD_SIZE;
+    let continuation_table_size =
+        size_of::<Array>().to_bytes().as_usize() + continuation_table.len() * WORD_SIZE;
 
     {
         let mut heap_offset = continuation_table_offset;
@@ -499,6 +519,32 @@ fn create_dynamic_heap(
         }
     }
 
+    // Add region0
+    let region0_offset = continuation_table_offset + continuation_table_size;
+    {
+        let mut heap_offset = region0_offset;
+        let region0_address = u32::try_from(heap_start + heap_offset).unwrap();
+        write_word(dynamic_heap, heap_offset, TAG_REGION);
+        heap_offset += WORD_SIZE;
+
+        if incremental {
+            write_word(dynamic_heap, heap_offset, make_pointer(region0_address));
+            heap_offset += WORD_SIZE;
+        }
+
+        // lower part of region id
+        write_word(dynamic_heap, heap_offset, 0);
+        heap_offset += WORD_SIZE;
+        // upper part of region id
+        write_word(dynamic_heap, heap_offset, 0);
+        heap_offset += WORD_SIZE;
+        // zero pages
+        write_word(dynamic_heap, heap_offset, 0);
+        heap_offset += WORD_SIZE;
+        // Simplification: Skip the vector pages blob
+        write_word(dynamic_heap, heap_offset, make_scalar(0));
+    }
+
     object_addrs
 }
 
@@ -510,6 +556,8 @@ fn create_static_heap(
     object_addrs: &FxHashMap<ObjectIdx, usize>,
     continuation_table_ptr_offset: usize,
     continuation_table_offset: usize,
+    region0_ptr_offset: usize,
+    region0_offset: usize,
     heap: &mut [u8],
 ) {
     let incremental = cfg!(feature = "incremental_gc");
@@ -568,11 +616,15 @@ fn create_static_heap(
         assert_eq!(offset, mutbox_offset);
     }
 
-    // Write continuation table pointer as the last word in static heap
+    // Write continuation table pointer as the second last word in static heap
     let continuation_table_ptr = continuation_table_offset as u32 + heap.as_ptr() as u32;
     write_word(
         heap,
         continuation_table_ptr_offset,
         make_pointer(continuation_table_ptr),
     );
+
+    // Write continuation table pointer as the second last word in static heap
+    let region0_ptr = region0_offset as u32 + heap.as_ptr() as u32;
+    write_word(heap, region0_ptr_offset, make_pointer(region0_ptr));
 }