Merge Preparation: Merge Precise Object Tagging to Graph Copy EOP

rts/motoko-rts-tests/src/continuation_table.rs

@@ -6,7 +6,7 @@ use motoko_rts::continuation_table::{
     continuation_count, recall_continuation, remember_continuation,
 };
 use motoko_rts::memory::alloc_blob;
-use motoko_rts::types::{Bytes, Value};
+use motoko_rts::types::{Bytes, Value, TAG_BLOB_B};
 
 pub unsafe fn test() {
     println!("Testing continuation table ...");
@@ -17,7 +17,7 @@ pub unsafe fn test() {
 
     let mut heap = initialize_test_memory();
 
-    let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, Bytes(0)));
+    let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, TAG_BLOB_B, Bytes(0)));
 
     let mut references: [usize; N] = [0; N];
     for i in 0..N {

rts/motoko-rts-tests/src/crc32.rs

@@ -1,7 +1,7 @@
 use crate::memory::{initialize_test_memory, reset_test_memory};
 
-use motoko_rts::principal_id::{base32_of_checksummed_blob, base32_to_blob};
-use motoko_rts::text::{text_compare, text_of_ptr_size};
+use motoko_rts::principal_id::{base32_of_checksummed_blob, base32_to_blob, blob_of_ptr_size};
+use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size};
 use motoko_rts::types::Bytes;
 
 pub unsafe fn test() {
@@ -41,36 +41,36 @@ pub unsafe fn test() {
 
     let text = text_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0));
     assert_eq!(
-        text_compare(
+        blob_compare(
             base32_to_blob(&mut heap, text),
-            text_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0))
+            blob_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0))
         ),
         0
     );
 
     let text = text_of_ptr_size(&mut heap, b"GEZDGNBVGY3TQOI".as_ptr(), Bytes(15));
     assert_eq!(
-        text_compare(
+        blob_compare(
             base32_to_blob(&mut heap, text),
-            text_of_ptr_size(&mut heap, b"123456789".as_ptr(), Bytes(9))
+            blob_of_ptr_size(&mut heap, b"123456789".as_ptr(), Bytes(9))
         ),
         0
     );
 
     let text = text_of_ptr_size(&mut heap, b"MFRGGZDFMZTWQ2LKNNWG23TPOA".as_ptr(), Bytes(26));
     assert_eq!(
-        text_compare(
+        blob_compare(
             base32_to_blob(&mut heap, text),
-            text_of_ptr_size(&mut heap, b"abcdefghijklmnop".as_ptr(), Bytes(16))
+            blob_of_ptr_size(&mut heap, b"abcdefghijklmnop".as_ptr(), Bytes(16))
         ),
         0
     );
 
     let text = text_of_ptr_size(&mut heap, b"em77e-bvlzu-aq".as_ptr(), Bytes(14));
     assert_eq!(
-        text_compare(
+        blob_compare(
             base32_to_blob(&mut heap, text),
-            text_of_ptr_size(
+            blob_of_ptr_size(
                 &mut heap,
                 b"\x23\x3f\xf2\x06\xab\xcd\x01".as_ptr(),
                 Bytes(7)

rts/motoko-rts-tests/src/gc.rs

@@ -275,7 +275,7 @@ pub fn check_dynamic_heap(
             if tag == TAG_MUTBOX {
                 // MutBoxes of static root array, will be scanned indirectly when checking the static root array.
                 offset += WORD_SIZE;
-            } else if tag == TAG_BLOB {
+            } else if tag == TAG_BLOB_B {
                 assert!(!is_forwarded);
                 // in-heap mark stack blobs
                 let length = read_word(heap, offset);
@@ -290,7 +290,7 @@ pub fn check_dynamic_heap(
             } else if mode == CheckMode::Stabilzation && tag == TAG_MUTBOX {
                 offset += WORD_SIZE;
             } else {
-                assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
+                assert!(is_array_or_slice_tag(tag));
 
                 if is_forwarded {
                     let forward_offset = forward - heap.as_ptr() as usize;
@@ -442,7 +442,7 @@ fn compute_reachable_objects(
 
 fn check_static_root_array(mut offset: usize, roots: &[ObjectIdx], heap: &[u8]) {
     let array_address = heap.as_ptr() as usize + offset;
-    assert_eq!(read_word(heap, offset), TAG_ARRAY);
+    assert_eq!(read_word(heap, offset), TAG_ARRAY_M);
     offset += WORD_SIZE;
 
     assert_eq!(read_word(heap, offset), make_pointer(array_address));
@@ -476,7 +476,7 @@ fn read_mutbox_field(mutbox_address: usize, heap: &[u8]) -> usize {
 
 fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx], heap: &[u8]) {
     let table_addr = heap.as_ptr() as usize + offset;
-    assert_eq!(read_word(heap, offset), TAG_ARRAY);
+    assert_eq!(read_word(heap, offset), TAG_ARRAY_M);
     offset += WORD_SIZE;
 
     assert_eq!(read_word(heap, offset), make_pointer(table_addr));
@@ -496,7 +496,7 @@ fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx],
 
 fn read_object_id(object_address: usize, heap: &[u8]) -> ObjectIdx {
     let tag = read_word(heap, object_address - heap.as_ptr() as usize);
-    assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
+    assert!(is_array_or_slice_tag(tag));
 
     // Skip object header for idx
     let idx_address = object_address + size_of::<Array>().to_bytes().as_usize();

rts/motoko-rts-tests/src/gc/heap.rs

@@ -353,7 +353,7 @@ fn create_dynamic_heap(
 
             // Store object header
             let address = heap_start + heap_offset;
-            write_word(dynamic_heap, heap_offset, TAG_ARRAY);
+            write_word(dynamic_heap, heap_offset, TAG_ARRAY_M);
             heap_offset += WORD_SIZE;
 
             write_word(dynamic_heap, heap_offset, make_pointer(address)); // forwarding pointer
@@ -413,7 +413,7 @@ fn create_dynamic_heap(
     }
     let static_root_array_address = heap_start + heap_offset;
     {
-        write_word(dynamic_heap, heap_offset, TAG_ARRAY);
+        write_word(dynamic_heap, heap_offset, TAG_ARRAY_M);
         heap_offset += WORD_SIZE;
 
         write_word(
@@ -435,7 +435,7 @@ fn create_dynamic_heap(
 
     let continuation_table_address = heap_start + heap_offset;
     {
-        write_word(dynamic_heap, heap_offset, TAG_ARRAY);
+        write_word(dynamic_heap, heap_offset, TAG_ARRAY_M);
         heap_offset += WORD_SIZE;
 
         write_word(

rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs

@@ -1,33 +1,37 @@
 use motoko_rts::{
     gc::incremental::array_slicing::slice_array,
     memory::alloc_array,
-    types::{Words, TAG_ARRAY, TAG_ARRAY_SLICE_MIN},
+    types::{Tag, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S, TAG_ARRAY_SLICE_MIN, TAG_ARRAY_T},
 };
 
 use crate::memory::TestMemory;
 
 pub unsafe fn test() {
     println!("  Testing array slicing...");
 
-    let mut mem = TestMemory::new(Words(1024 * 1024));
-    // multiple of slice increment
-    test_array_slicing(&mut mem, 4096);
-    // odd remainder of slice increment
-    test_array_slicing(&mut mem, 3999);
-    // small array
-    test_array_slicing(&mut mem, 10);
-    // empty array
-    test_array_slicing(&mut mem, 0);
+    let tags = vec![TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T, TAG_ARRAY_S];
+
+    for tag in tags.into_iter() {
+        let mut mem = TestMemory::new(Words(1024 * 1024));
+        // multiple of slice increment
+        test_array_slicing(&mut mem, tag, 4096);
+        // odd remainder of slice increment
+        test_array_slicing(&mut mem, tag, 3999);
+        // small array
+        test_array_slicing(&mut mem, tag, 10);
+        // empty array
+        test_array_slicing(&mut mem, tag, 0);
+    }
 }
 
-unsafe fn test_array_slicing(mem: &mut TestMemory, array_length: usize) {
-    let array = alloc_array(mem, array_length).as_array();
+unsafe fn test_array_slicing(mem: &mut TestMemory, tag: Tag, array_length: usize) {
+    let array = alloc_array(mem, tag, array_length).as_array();
     let mut last_offset = 0;
     loop {
         let new_offset = slice_array(array);
         assert!(new_offset > last_offset || array.len() == 0 && new_offset == 0);
         last_offset = new_offset;
-        if (*array).header.tag == TAG_ARRAY {
+        if (*array).header.tag == tag {
             break;
         }
         assert!((*array).header.tag >= TAG_ARRAY_SLICE_MIN);

rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs

@@ -15,7 +15,10 @@ use motoko_rts::{
         IncrementalGC,
     },
     memory::{alloc_array, alloc_blob, Memory},
-    types::{Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY, TAG_BLOB},
+    types::{
+        Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S,
+        TAG_ARRAY_T, TAG_BLOB_A, TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T,
+    },
 };
 
 use crate::{gc::utils::WORD_SIZE, memory::TestMemory};
@@ -322,7 +325,7 @@ unsafe fn iterate_large_partition(
     let mut time = BoundedTime::new(0);
     while iterator.has_object() {
         let object = iterator.current_object();
-        assert_eq!(object.tag(), TAG_BLOB);
+        assert_eq!(object.tag(), TAG_BLOB_B);
         let size = block_size(object as *const Tag);
         detected_sizes.push(size);
         time.tick();
@@ -413,7 +416,7 @@ impl PartitionedTestHeap {
 
     pub fn allocate_array(&mut self, elements: &[Value]) -> Value {
         unsafe {
-            let array = alloc_array(self, elements.len());
+            let array = alloc_array(self, TAG_ARRAY_M, elements.len());
             for index in 0..elements.len() {
                 let raw_array = array.as_array();
                 raw_array.set(index, elements[index], self);
@@ -423,14 +426,18 @@ impl PartitionedTestHeap {
     }
 
     pub fn allocate_blob(&mut self, size: usize) -> Value {
-        unsafe { alloc_blob(self, Bytes(size)) }
+        unsafe { alloc_blob(self, TAG_BLOB_B, Bytes(size)) }
     }
 }
 
 unsafe fn block_size(block: *const Tag) -> usize {
     match *block {
-        TAG_ARRAY => size_of::<Array>() + (block as *const Array).len() * WORD_SIZE,
-        TAG_BLOB => size_of::<Blob>() + (block as *const Blob).len().as_usize(),
+        TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S => {
+            size_of::<Array>() + (block as *const Array).len() * WORD_SIZE
+        }
+        TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P | TAG_BLOB_A => {
+            size_of::<Blob>() + (block as *const Blob).len().as_usize()
+        }
         _ => unimplemented!(),
     }
 }

rts/motoko-rts-tests/src/principal_id.rs

@@ -1,7 +1,9 @@
 use crate::memory::{initialize_test_memory, reset_test_memory};
 
-use motoko_rts::principal_id::{blob_of_principal, principal_of_blob};
-use motoko_rts::text::{text_compare, text_of_ptr_size, text_of_str};
+use motoko_rts::principal_id::{
+    blob_of_principal, blob_of_ptr_size, blob_of_str, principal_of_blob,
+};
+use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size, text_of_str};
 use motoko_rts::types::Bytes;
 
 pub unsafe fn test() {
@@ -36,13 +38,13 @@ pub unsafe fn test() {
 
     let text = text_of_str(&mut heap, "aaaaa-aa");
     let principal = blob_of_principal(&mut heap, text);
-    assert_eq!(text_compare(principal, text_of_str(&mut heap, ""),), 0);
+    assert_eq!(blob_compare(principal, blob_of_str(&mut heap, ""),), 0);
 
     let text = text_of_str(&mut heap, "bfozs-kwa73-7nadi");
     assert_eq!(
-        text_compare(
+        blob_compare(
             blob_of_principal(&mut heap, text),
-            text_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5))
+            blob_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5))
         ),
         0
     );

rts/motoko-rts-tests/src/stabilization.rs

@@ -16,7 +16,7 @@ use motoko_rts::{
     stabilization::{
         deserialization::Deserialization, graph_copy::GraphCopy, serialization::Serialization,
     },
-    types::{Value, Words},
+    types::{Value, Words, TAG_ARRAY_M},
 };
 use oorandom::Rand32;
 
@@ -91,7 +91,7 @@ impl RandomHeap {
     fn clear_continuation_table(&mut self) {
         let table_pointer = self.memory.continuation_table_variable_address() as *mut Value;
         unsafe {
-            *table_pointer = alloc_array(&mut self.memory, 0);
+            *table_pointer = alloc_array(&mut self.memory, TAG_ARRAY_M, 0);
         }
     }
 

rts/motoko-rts-tests/src/stabilization/layout.rs

@@ -8,11 +8,19 @@ pub unsafe fn test() {
 
 fn test_stable_tags() {
     for object_kind in [
-        StableObjectKind::Array,
+        StableObjectKind::ArrayImmutable,
+        StableObjectKind::ArrayMutable,
+        StableObjectKind::ArrayTuple,
+        StableObjectKind::ArraySharedFunction,
         StableObjectKind::MutBox,
         StableObjectKind::Object,
-        StableObjectKind::Blob,
-        StableObjectKind::Bits64,
+        StableObjectKind::BlobBytes,
+        StableObjectKind::BlobText,
+        StableObjectKind::BlobPrincipal,
+        StableObjectKind::BlobActor,
+        StableObjectKind::Bits64Unsigned,
+        StableObjectKind::Bits64Signed,
+        StableObjectKind::Bits64Float,
         StableObjectKind::Region,
         StableObjectKind::Variant,
         StableObjectKind::Concat,

rts/motoko-rts-tests/src/text.rs

@@ -8,7 +8,7 @@ use motoko_rts::text::{
     text_singleton, text_size,
 };
 use motoko_rts::text_iter::{text_iter, text_iter_done, text_iter_next};
-use motoko_rts::types::{Bytes, Value, TAG_BLOB};
+use motoko_rts::types::{Bytes, Value, TAG_BLOB_T};
 
 use std::convert::TryFrom;
 
@@ -66,7 +66,7 @@ pub unsafe fn test() {
     for i in 0..8 {
         let str = &STR[0..i + 1];
         let text = text_of_str(&mut mem, str);
-        assert_eq!(text.tag(), TAG_BLOB);
+        assert_eq!(text.tag(), TAG_BLOB_T);
         let iter = TextIter::from_text(&mut mem, text);
         assert_eq!(iter.collect::<String>(), str);
     }

rts/motoko-rts/src/allocator.rs

@@ -3,7 +3,7 @@
 use alloc::alloc::{GlobalAlloc, Layout};
 //use core::ptr::null_mut;
 use crate::memory::{alloc_blob, ic};
-use crate::types::Bytes;
+use crate::types::{Bytes, TAG_BLOB_B};
 
 pub struct EphemeralAllocator;
 
@@ -23,7 +23,8 @@ unsafe impl GlobalAlloc for EphemeralAllocator {
         let word_size = crate::constants::WORD_SIZE;
         let min_align = (align + word_size - 1) / word_size * word_size;
         let blob_size = size + min_align - word_size;
-        let blob = alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, Bytes(blob_size)).as_blob_mut();
+        let blob = alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, TAG_BLOB_B, Bytes(blob_size))
+            .as_blob_mut();
         let payload_address = blob.payload_addr() as usize;
         let aligned_address = (payload_address + min_align - 1) / min_align * min_align;
 

rts/motoko-rts/src/blob_iter.rs

@@ -1,7 +1,7 @@
 use crate::{
     barriers::allocation_barrier,
     memory::Memory,
-    types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY},
+    types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY_T},
 };
 
 use motoko_rts_macros::ic_mem_fn;
@@ -17,7 +17,7 @@ unsafe fn blob_iter<M: crate::memory::Memory>(mem: &mut M, blob: Value) -> Value
 
     // NB. cannot use as_array() here as we didn't write the header yet
     let iter_array = iter_ptr.get_ptr() as *mut Array;
-    (*iter_array).header.tag = TAG_ARRAY;
+    (*iter_array).header.tag = TAG_ARRAY_T;
     (*iter_array).header.init_forward(iter_ptr);
     (*iter_array).len = 2;
 

rts/motoko-rts/src/continuation_table.rs

@@ -30,7 +30,7 @@
 use crate::barriers::{allocation_barrier, write_with_barrier};
 use crate::memory::{alloc_array, Memory};
 use crate::rts_trap_with;
-use crate::types::Value;
+use crate::types::{Value, TAG_ARRAY_M};
 
 use motoko_rts_macros::ic_mem_fn;
 
@@ -50,7 +50,7 @@ static mut N_CONTINUATIONS: usize = 0;
 static mut FREE_SLOT: usize = 0;
 
 unsafe fn create_continuation_table<M: Memory>(mem: &mut M) {
-    TABLE = alloc_array(mem, INITIAL_SIZE);
+    TABLE = alloc_array(mem, TAG_ARRAY_M, INITIAL_SIZE);
     FREE_SLOT = 0;
     N_CONTINUATIONS = 0;
 
@@ -69,7 +69,7 @@ unsafe fn double_continuation_table<M: Memory>(mem: &mut M) {
 
     let new_size = old_size * 2;
 
-    let new_table = alloc_array(mem, new_size);
+    let new_table = alloc_array(mem, TAG_ARRAY_M, new_size);
     let new_array = new_table.as_array();
 
     for i in 0..old_size {