diff --git a/rts/motoko-rts-tests/src/continuation_table.rs b/rts/motoko-rts-tests/src/continuation_table.rs index 18a13635c2d..9f83fa28f64 100644 --- a/rts/motoko-rts-tests/src/continuation_table.rs +++ b/rts/motoko-rts-tests/src/continuation_table.rs @@ -6,7 +6,7 @@ use motoko_rts::continuation_table::{ continuation_count, recall_continuation, remember_continuation, }; use motoko_rts::memory::alloc_blob; -use motoko_rts::types::{Bytes, Value}; +use motoko_rts::types::{Bytes, Value, TAG_BLOB_B}; pub unsafe fn test() { println!("Testing continuation table ..."); @@ -17,7 +17,7 @@ pub unsafe fn test() { let mut heap = initialize_test_memory(); - let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, Bytes(0))); + let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, TAG_BLOB_B, Bytes(0))); let mut references: [usize; N] = [0; N]; for i in 0..N { diff --git a/rts/motoko-rts-tests/src/crc32.rs b/rts/motoko-rts-tests/src/crc32.rs index dc40f8352e8..b76439cf7a9 100644 --- a/rts/motoko-rts-tests/src/crc32.rs +++ b/rts/motoko-rts-tests/src/crc32.rs @@ -1,7 +1,7 @@ use crate::memory::{initialize_test_memory, reset_test_memory}; -use motoko_rts::principal_id::{base32_of_checksummed_blob, base32_to_blob}; -use motoko_rts::text::{text_compare, text_of_ptr_size}; +use motoko_rts::principal_id::{base32_of_checksummed_blob, base32_to_blob, blob_of_ptr_size}; +use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size}; use motoko_rts::types::Bytes; pub unsafe fn test() { @@ -41,36 +41,36 @@ pub unsafe fn test() { let text = text_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0)); assert_eq!( - text_compare( + blob_compare( base32_to_blob(&mut heap, text), - text_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0)) + blob_of_ptr_size(&mut heap, b"".as_ptr(), Bytes(0)) ), 0 ); let text = text_of_ptr_size(&mut heap, b"GEZDGNBVGY3TQOI".as_ptr(), Bytes(15)); assert_eq!( - text_compare( + blob_compare( base32_to_blob(&mut heap, text), - text_of_ptr_size(&mut heap, b"123456789".as_ptr(), Bytes(9)) + blob_of_ptr_size(&mut heap, b"123456789".as_ptr(), Bytes(9)) ), 0 ); let text = text_of_ptr_size(&mut heap, b"MFRGGZDFMZTWQ2LKNNWG23TPOA".as_ptr(), Bytes(26)); assert_eq!( - text_compare( + blob_compare( base32_to_blob(&mut heap, text), - text_of_ptr_size(&mut heap, b"abcdefghijklmnop".as_ptr(), Bytes(16)) + blob_of_ptr_size(&mut heap, b"abcdefghijklmnop".as_ptr(), Bytes(16)) ), 0 ); let text = text_of_ptr_size(&mut heap, b"em77e-bvlzu-aq".as_ptr(), Bytes(14)); assert_eq!( - text_compare( + blob_compare( base32_to_blob(&mut heap, text), - text_of_ptr_size( + blob_of_ptr_size( &mut heap, b"\x23\x3f\xf2\x06\xab\xcd\x01".as_ptr(), Bytes(7) diff --git a/rts/motoko-rts-tests/src/gc.rs b/rts/motoko-rts-tests/src/gc.rs index 639cbf7dd22..5b7001d0f52 100644 --- a/rts/motoko-rts-tests/src/gc.rs +++ b/rts/motoko-rts-tests/src/gc.rs @@ -275,7 +275,7 @@ pub fn check_dynamic_heap( if tag == TAG_MUTBOX { // MutBoxes of static root array, will be scanned indirectly when checking the static root array. offset += WORD_SIZE; - } else if tag == TAG_BLOB { + } else if tag == TAG_BLOB_B { assert!(!is_forwarded); // in-heap mark stack blobs let length = read_word(heap, offset); @@ -290,7 +290,7 @@ pub fn check_dynamic_heap( } else if mode == CheckMode::Stabilzation && tag == TAG_MUTBOX { offset += WORD_SIZE; } else { - assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN); + assert!(is_array_or_slice_tag(tag)); if is_forwarded { let forward_offset = forward - heap.as_ptr() as usize; @@ -442,7 +442,7 @@ fn compute_reachable_objects( fn check_static_root_array(mut offset: usize, roots: &[ObjectIdx], heap: &[u8]) { let array_address = heap.as_ptr() as usize + offset; - assert_eq!(read_word(heap, offset), TAG_ARRAY); + assert_eq!(read_word(heap, offset), TAG_ARRAY_M); offset += WORD_SIZE; assert_eq!(read_word(heap, offset), make_pointer(array_address)); @@ -476,7 +476,7 @@ fn read_mutbox_field(mutbox_address: usize, heap: &[u8]) -> usize { fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx], heap: &[u8]) { let table_addr = heap.as_ptr() as usize + offset; - assert_eq!(read_word(heap, offset), TAG_ARRAY); + assert_eq!(read_word(heap, offset), TAG_ARRAY_M); offset += WORD_SIZE; assert_eq!(read_word(heap, offset), make_pointer(table_addr)); @@ -496,7 +496,7 @@ fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx], fn read_object_id(object_address: usize, heap: &[u8]) -> ObjectIdx { let tag = read_word(heap, object_address - heap.as_ptr() as usize); - assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN); + assert!(is_array_or_slice_tag(tag)); // Skip object header for idx let idx_address = object_address + size_of::().to_bytes().as_usize(); diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index 99e4459d095..219ae719d39 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -353,7 +353,7 @@ fn create_dynamic_heap( // Store object header let address = heap_start + heap_offset; - write_word(dynamic_heap, heap_offset, TAG_ARRAY); + write_word(dynamic_heap, heap_offset, TAG_ARRAY_M); heap_offset += WORD_SIZE; write_word(dynamic_heap, heap_offset, make_pointer(address)); // forwarding pointer @@ -413,7 +413,7 @@ fn create_dynamic_heap( } let static_root_array_address = heap_start + heap_offset; { - write_word(dynamic_heap, heap_offset, TAG_ARRAY); + write_word(dynamic_heap, heap_offset, TAG_ARRAY_M); heap_offset += WORD_SIZE; write_word( @@ -435,7 +435,7 @@ fn create_dynamic_heap( let continuation_table_address = heap_start + heap_offset; { - write_word(dynamic_heap, heap_offset, TAG_ARRAY); + write_word(dynamic_heap, heap_offset, TAG_ARRAY_M); heap_offset += WORD_SIZE; write_word( diff --git a/rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs b/rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs index bf01fa11bd5..df2060a222c 100644 --- a/rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs +++ b/rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs @@ -1,7 +1,7 @@ use motoko_rts::{ gc::incremental::array_slicing::slice_array, memory::alloc_array, - types::{Words, TAG_ARRAY, TAG_ARRAY_SLICE_MIN}, + types::{Tag, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S, TAG_ARRAY_SLICE_MIN, TAG_ARRAY_T}, }; use crate::memory::TestMemory; @@ -9,25 +9,29 @@ use crate::memory::TestMemory; pub unsafe fn test() { println!(" Testing array slicing..."); - let mut mem = TestMemory::new(Words(1024 * 1024)); - // multiple of slice increment - test_array_slicing(&mut mem, 4096); - // odd remainder of slice increment - test_array_slicing(&mut mem, 3999); - // small array - test_array_slicing(&mut mem, 10); - // empty array - test_array_slicing(&mut mem, 0); + let tags = vec![TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T, TAG_ARRAY_S]; + + for tag in tags.into_iter() { + let mut mem = TestMemory::new(Words(1024 * 1024)); + // multiple of slice increment + test_array_slicing(&mut mem, tag, 4096); + // odd remainder of slice increment + test_array_slicing(&mut mem, tag, 3999); + // small array + test_array_slicing(&mut mem, tag, 10); + // empty array + test_array_slicing(&mut mem, tag, 0); + } } -unsafe fn test_array_slicing(mem: &mut TestMemory, array_length: usize) { - let array = alloc_array(mem, array_length).as_array(); +unsafe fn test_array_slicing(mem: &mut TestMemory, tag: Tag, array_length: usize) { + let array = alloc_array(mem, tag, array_length).as_array(); let mut last_offset = 0; loop { let new_offset = slice_array(array); assert!(new_offset > last_offset || array.len() == 0 && new_offset == 0); last_offset = new_offset; - if (*array).header.tag == TAG_ARRAY { + if (*array).header.tag == tag { break; } assert!((*array).header.tag >= TAG_ARRAY_SLICE_MIN); diff --git a/rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs b/rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs index f68a09e896c..c31d737a1e8 100644 --- a/rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs +++ b/rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs @@ -15,7 +15,10 @@ use motoko_rts::{ IncrementalGC, }, memory::{alloc_array, alloc_blob, Memory}, - types::{Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY, TAG_BLOB}, + types::{ + Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S, + TAG_ARRAY_T, TAG_BLOB_A, TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T, + }, }; use crate::{gc::utils::WORD_SIZE, memory::TestMemory}; @@ -322,7 +325,7 @@ unsafe fn iterate_large_partition( let mut time = BoundedTime::new(0); while iterator.has_object() { let object = iterator.current_object(); - assert_eq!(object.tag(), TAG_BLOB); + assert_eq!(object.tag(), TAG_BLOB_B); let size = block_size(object as *const Tag); detected_sizes.push(size); time.tick(); @@ -413,7 +416,7 @@ impl PartitionedTestHeap { pub fn allocate_array(&mut self, elements: &[Value]) -> Value { unsafe { - let array = alloc_array(self, elements.len()); + let array = alloc_array(self, TAG_ARRAY_M, elements.len()); for index in 0..elements.len() { let raw_array = array.as_array(); raw_array.set(index, elements[index], self); @@ -423,14 +426,18 @@ impl PartitionedTestHeap { } pub fn allocate_blob(&mut self, size: usize) -> Value { - unsafe { alloc_blob(self, Bytes(size)) } + unsafe { alloc_blob(self, TAG_BLOB_B, Bytes(size)) } } } unsafe fn block_size(block: *const Tag) -> usize { match *block { - TAG_ARRAY => size_of::() + (block as *const Array).len() * WORD_SIZE, - TAG_BLOB => size_of::() + (block as *const Blob).len().as_usize(), + TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S => { + size_of::() + (block as *const Array).len() * WORD_SIZE + } + TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P | TAG_BLOB_A => { + size_of::() + (block as *const Blob).len().as_usize() + } _ => unimplemented!(), } } diff --git a/rts/motoko-rts-tests/src/principal_id.rs b/rts/motoko-rts-tests/src/principal_id.rs index 8c9a8d3eb99..2cb8b4d000c 100644 --- a/rts/motoko-rts-tests/src/principal_id.rs +++ b/rts/motoko-rts-tests/src/principal_id.rs @@ -1,7 +1,9 @@ use crate::memory::{initialize_test_memory, reset_test_memory}; -use motoko_rts::principal_id::{blob_of_principal, principal_of_blob}; -use motoko_rts::text::{text_compare, text_of_ptr_size, text_of_str}; +use motoko_rts::principal_id::{ + blob_of_principal, blob_of_ptr_size, blob_of_str, principal_of_blob, +}; +use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size, text_of_str}; use motoko_rts::types::Bytes; pub unsafe fn test() { @@ -36,13 +38,13 @@ pub unsafe fn test() { let text = text_of_str(&mut heap, "aaaaa-aa"); let principal = blob_of_principal(&mut heap, text); - assert_eq!(text_compare(principal, text_of_str(&mut heap, ""),), 0); + assert_eq!(blob_compare(principal, blob_of_str(&mut heap, ""),), 0); let text = text_of_str(&mut heap, "bfozs-kwa73-7nadi"); assert_eq!( - text_compare( + blob_compare( blob_of_principal(&mut heap, text), - text_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5)) + blob_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5)) ), 0 ); diff --git a/rts/motoko-rts-tests/src/stabilization.rs b/rts/motoko-rts-tests/src/stabilization.rs index 9e9d0a10279..b50e89b8e23 100644 --- a/rts/motoko-rts-tests/src/stabilization.rs +++ b/rts/motoko-rts-tests/src/stabilization.rs @@ -16,7 +16,7 @@ use motoko_rts::{ stabilization::{ deserialization::Deserialization, graph_copy::GraphCopy, serialization::Serialization, }, - types::{Value, Words}, + types::{Value, Words, TAG_ARRAY_M}, }; use oorandom::Rand32; @@ -91,7 +91,7 @@ impl RandomHeap { fn clear_continuation_table(&mut self) { let table_pointer = self.memory.continuation_table_variable_address() as *mut Value; unsafe { - *table_pointer = alloc_array(&mut self.memory, 0); + *table_pointer = alloc_array(&mut self.memory, TAG_ARRAY_M, 0); } } diff --git a/rts/motoko-rts-tests/src/stabilization/layout.rs b/rts/motoko-rts-tests/src/stabilization/layout.rs index 04de16433e4..0400af10045 100644 --- a/rts/motoko-rts-tests/src/stabilization/layout.rs +++ b/rts/motoko-rts-tests/src/stabilization/layout.rs @@ -8,11 +8,19 @@ pub unsafe fn test() { fn test_stable_tags() { for object_kind in [ - StableObjectKind::Array, + StableObjectKind::ArrayImmutable, + StableObjectKind::ArrayMutable, + StableObjectKind::ArrayTuple, + StableObjectKind::ArraySharedFunction, StableObjectKind::MutBox, StableObjectKind::Object, - StableObjectKind::Blob, - StableObjectKind::Bits64, + StableObjectKind::BlobBytes, + StableObjectKind::BlobText, + StableObjectKind::BlobPrincipal, + StableObjectKind::BlobActor, + StableObjectKind::Bits64Unsigned, + StableObjectKind::Bits64Signed, + StableObjectKind::Bits64Float, StableObjectKind::Region, StableObjectKind::Variant, StableObjectKind::Concat, diff --git a/rts/motoko-rts-tests/src/text.rs b/rts/motoko-rts-tests/src/text.rs index 5216dfac4d0..218d265a5d2 100644 --- a/rts/motoko-rts-tests/src/text.rs +++ b/rts/motoko-rts-tests/src/text.rs @@ -8,7 +8,7 @@ use motoko_rts::text::{ text_singleton, text_size, }; use motoko_rts::text_iter::{text_iter, text_iter_done, text_iter_next}; -use motoko_rts::types::{Bytes, Value, TAG_BLOB}; +use motoko_rts::types::{Bytes, Value, TAG_BLOB_T}; use std::convert::TryFrom; @@ -66,7 +66,7 @@ pub unsafe fn test() { for i in 0..8 { let str = &STR[0..i + 1]; let text = text_of_str(&mut mem, str); - assert_eq!(text.tag(), TAG_BLOB); + assert_eq!(text.tag(), TAG_BLOB_T); let iter = TextIter::from_text(&mut mem, text); assert_eq!(iter.collect::(), str); } diff --git a/rts/motoko-rts/src/allocator.rs b/rts/motoko-rts/src/allocator.rs index a163a8bc939..493d850c247 100644 --- a/rts/motoko-rts/src/allocator.rs +++ b/rts/motoko-rts/src/allocator.rs @@ -3,7 +3,7 @@ use alloc::alloc::{GlobalAlloc, Layout}; //use core::ptr::null_mut; use crate::memory::{alloc_blob, ic}; -use crate::types::Bytes; +use crate::types::{Bytes, TAG_BLOB_B}; pub struct EphemeralAllocator; @@ -23,7 +23,8 @@ unsafe impl GlobalAlloc for EphemeralAllocator { let word_size = crate::constants::WORD_SIZE; let min_align = (align + word_size - 1) / word_size * word_size; let blob_size = size + min_align - word_size; - let blob = alloc_blob::(&mut ic::IcMemory, Bytes(blob_size)).as_blob_mut(); + let blob = alloc_blob::(&mut ic::IcMemory, TAG_BLOB_B, Bytes(blob_size)) + .as_blob_mut(); let payload_address = blob.payload_addr() as usize; let aligned_address = (payload_address + min_align - 1) / min_align * min_align; diff --git a/rts/motoko-rts/src/blob_iter.rs b/rts/motoko-rts/src/blob_iter.rs index 5b1a008b52a..85f9ea94b7a 100644 --- a/rts/motoko-rts/src/blob_iter.rs +++ b/rts/motoko-rts/src/blob_iter.rs @@ -1,7 +1,7 @@ use crate::{ barriers::allocation_barrier, memory::Memory, - types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY}, + types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY_T}, }; use motoko_rts_macros::ic_mem_fn; @@ -17,7 +17,7 @@ unsafe fn blob_iter(mem: &mut M, blob: Value) -> Value // NB. cannot use as_array() here as we didn't write the header yet let iter_array = iter_ptr.get_ptr() as *mut Array; - (*iter_array).header.tag = TAG_ARRAY; + (*iter_array).header.tag = TAG_ARRAY_T; (*iter_array).header.init_forward(iter_ptr); (*iter_array).len = 2; diff --git a/rts/motoko-rts/src/continuation_table.rs b/rts/motoko-rts/src/continuation_table.rs index 7ec02640df1..316ef8105ba 100644 --- a/rts/motoko-rts/src/continuation_table.rs +++ b/rts/motoko-rts/src/continuation_table.rs @@ -30,7 +30,7 @@ use crate::barriers::{allocation_barrier, write_with_barrier}; use crate::memory::{alloc_array, Memory}; use crate::rts_trap_with; -use crate::types::Value; +use crate::types::{Value, TAG_ARRAY_M}; use motoko_rts_macros::ic_mem_fn; @@ -50,7 +50,7 @@ static mut N_CONTINUATIONS: usize = 0; static mut FREE_SLOT: usize = 0; unsafe fn create_continuation_table(mem: &mut M) { - TABLE = alloc_array(mem, INITIAL_SIZE); + TABLE = alloc_array(mem, TAG_ARRAY_M, INITIAL_SIZE); FREE_SLOT = 0; N_CONTINUATIONS = 0; @@ -69,7 +69,7 @@ unsafe fn double_continuation_table(mem: &mut M) { let new_size = old_size * 2; - let new_table = alloc_array(mem, new_size); + let new_table = alloc_array(mem, TAG_ARRAY_M, new_size); let new_array = new_table.as_array(); for i in 0..old_size { diff --git a/rts/motoko-rts/src/debug.rs b/rts/motoko-rts/src/debug.rs index ef59457e153..feb1c9a03f7 100644 --- a/rts/motoko-rts/src/debug.rs +++ b/rts/motoko-rts/src/debug.rs @@ -165,7 +165,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) { let obj_ind = obj as *const ObjInd; let _ = write!(buf, "", (*obj_ind).field.get_raw()); } - TAG_ARRAY | TAG_ARRAY_SLICE_MIN.. => { + TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S | TAG_ARRAY_SLICE_MIN.. => { let array = obj as *mut Array; let _ = write!(buf, ""); } } - TAG_BITS64 => { + TAG_BITS64_U | TAG_BITS64_S | TAG_BITS64_F => { let bits64 = obj as *const Bits64; let _ = write!(buf, "", (*bits64).bits); } @@ -204,7 +204,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) { (*variant).field.get_raw() ); } - TAG_BLOB => { + TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P | TAG_BLOB_A => { let blob = obj.as_blob(); let _ = write!(buf, "", blob.len().as_usize()); } diff --git a/rts/motoko-rts/src/gc/incremental/array_slicing.rs b/rts/motoko-rts/src/gc/incremental/array_slicing.rs index 222f2cbe678..d3aef1c28fc 100644 --- a/rts/motoko-rts/src/gc/incremental/array_slicing.rs +++ b/rts/motoko-rts/src/gc/incremental/array_slicing.rs @@ -8,14 +8,13 @@ use crate::types::*; pub unsafe fn slice_array(array: *mut Array) -> usize { const SLICE_INCREMENT: usize = 128; debug_assert!(SLICE_INCREMENT >= TAG_ARRAY_SLICE_MIN); - let tag = (*array).header.tag; - let slice_start = if tag >= TAG_ARRAY_SLICE_MIN { tag } else { 0 }; + let (base_tag, slice_start) = array.get_slice_start(); if array.len() - slice_start > SLICE_INCREMENT { let new_start = slice_start + SLICE_INCREMENT; - (*array).header.tag = new_start; + array.set_slice_start(base_tag, new_start); new_start } else { - (*array).header.tag = TAG_ARRAY; + array.restore_tag(base_tag); array.len() } } diff --git a/rts/motoko-rts/src/gc/incremental/mark_stack.rs b/rts/motoko-rts/src/gc/incremental/mark_stack.rs index 8e192e93598..2a8c4c7d20f 100644 --- a/rts/motoko-rts/src/gc/incremental/mark_stack.rs +++ b/rts/motoko-rts/src/gc/incremental/mark_stack.rs @@ -29,7 +29,7 @@ use core::ptr::null_mut; use crate::memory::{alloc_blob, Memory}; -use crate::types::{size_of, Blob, Value}; +use crate::types::{size_of, Blob, Value, TAG_BLOB_B}; #[repr(C)] pub struct MarkStack { @@ -97,8 +97,8 @@ impl MarkStack { unsafe fn new_table(mem: &mut M, previous: *mut StackTable) -> *mut StackTable { // No post allocation barrier as this RTS-internal blob will be collected by the GC. - let table = - alloc_blob(mem, size_of::().to_bytes()).as_blob_mut() as *mut StackTable; + let table = alloc_blob(mem, TAG_BLOB_B, size_of::().to_bytes()).as_blob_mut() + as *mut StackTable; (*table).previous = previous; (*table).next = null_mut(); if previous != null_mut() { diff --git a/rts/motoko-rts/src/gc/incremental/roots.rs b/rts/motoko-rts/src/gc/incremental/roots.rs index 9394209e256..3f746bd05b5 100644 --- a/rts/motoko-rts/src/gc/incremental/roots.rs +++ b/rts/motoko-rts/src/gc/incremental/roots.rs @@ -56,9 +56,9 @@ unsafe fn static_variables_location() -> *mut Value { pub unsafe fn initialize_static_variables(mem: &mut M, amount: usize) { use super::barriers::write_with_barrier; use crate::memory::alloc_array; - use crate::types::NULL_POINTER; + use crate::types::{NULL_POINTER, TAG_ARRAY_M}; - let variables = alloc_array(mem, amount); + let variables = alloc_array(mem, TAG_ARRAY_M, amount); let array = variables.as_array(); for index in 0..amount { array.initialize(index, NULL_POINTER, mem); diff --git a/rts/motoko-rts/src/gc/incremental/sanity_checks/remembered_set.rs b/rts/motoko-rts/src/gc/incremental/sanity_checks/remembered_set.rs index 36284843183..a88c5fd7d6e 100644 --- a/rts/motoko-rts/src/gc/incremental/sanity_checks/remembered_set.rs +++ b/rts/motoko-rts/src/gc/incremental/sanity_checks/remembered_set.rs @@ -38,7 +38,7 @@ use core::ptr::null_mut; use crate::constants::WORD_SIZE; use crate::memory::{alloc_blob, Memory}; -use crate::types::{block_size, Blob, Bytes, Value}; +use crate::types::{block_size, Blob, Bytes, Value, TAG_BLOB_B}; pub struct RememberedSet { hash_table: *mut Blob, @@ -224,7 +224,7 @@ impl RememberedSetIterator { unsafe fn new_table(mem: &mut M, size: usize) -> *mut Blob { // No post allocation barrier as this RTS-internal blob will be collected by the GC. - let table = alloc_blob(mem, Bytes(size * size_of::())).as_blob_mut(); + let table = alloc_blob(mem, TAG_BLOB_B, Bytes(size * size_of::())).as_blob_mut(); for index in 0..size { table_set(table, index, null_ptr_value()); } @@ -234,7 +234,8 @@ unsafe fn new_table(mem: &mut M, size: usize) -> *mut Blob { unsafe fn new_collision_node(mem: &mut M, value: Value) -> *mut CollisionNode { debug_assert!(!is_null_ptr_value(value)); // No post allocation barrier as this RTS-internal blob will be collected by the GC. - let node = alloc_blob(mem, Bytes(size_of::())).as_blob_mut() as *mut CollisionNode; + let node = alloc_blob(mem, TAG_BLOB_B, Bytes(size_of::())).as_blob_mut() + as *mut CollisionNode; (*node).entry = HashEntry { value, next_collision_ptr: null_mut(), diff --git a/rts/motoko-rts/src/idl.rs b/rts/motoko-rts/src/idl.rs index d4a07d61097..79b46612532 100644 --- a/rts/motoko-rts/src/idl.rs +++ b/rts/motoko-rts/src/idl.rs @@ -5,7 +5,7 @@ use crate::idl_trap_with; use crate::libc_declarations::{c_void, memcmp}; use crate::memory::{alloc_blob, Memory}; use crate::persistence::compatibility::TypeDescriptor; -use crate::types::{Value, Words}; +use crate::types::{Value, Words, TAG_BLOB_B}; use crate::utf8::utf8_validate; use core::cmp::min; @@ -133,7 +133,7 @@ unsafe fn parse_fields(mode: CompatibilityMode, buf: *mut Buf, n_types: u32) { // NB. This function assumes the allocation does not need to survive GC // Therefore, no post allocation barrier is applied. unsafe fn alloc(mem: &mut M, size: Words) -> *mut u8 { - alloc_blob(mem, size.to_bytes()) + alloc_blob(mem, TAG_BLOB_B, size.to_bytes()) .as_blob_mut() .payload_addr() } diff --git a/rts/motoko-rts/src/memory.rs b/rts/motoko-rts/src/memory.rs index e2263dce486..cb3e7b9dae5 100644 --- a/rts/motoko-rts/src/memory.rs +++ b/rts/motoko-rts/src/memory.rs @@ -34,11 +34,12 @@ pub trait Memory { /// Note: After initialization, the post allocation barrier needs to be applied to all mutator objects. /// For RTS-internal blobs that can be collected by the next GC run, the post allocation barrier can be omitted. #[ic_mem_fn] -pub unsafe fn alloc_blob(mem: &mut M, size: Bytes) -> Value { +pub unsafe fn alloc_blob(mem: &mut M, tag: Tag, size: Bytes) -> Value { + debug_assert!(is_blob_tag(tag)); let ptr = mem.alloc_words(size_of::() + size.to_words()); // NB. Cannot use `as_blob` here as we didn't write the header yet let blob = ptr.get_ptr() as *mut Blob; - (*blob).header.tag = TAG_BLOB; + (*blob).header.tag = tag; (*blob).header.init_forward(ptr); (*blob).len = size; @@ -48,13 +49,14 @@ pub unsafe fn alloc_blob(mem: &mut M, size: Bytes) -> Value { /// Allocate a new array. /// Note: After initialization, the post allocation barrier needs to be applied to all mutator objects. #[ic_mem_fn] -pub unsafe fn alloc_array(mem: &mut M, len: usize) -> Value { +pub unsafe fn alloc_array(mem: &mut M, tag: Tag, len: usize) -> Value { + debug_assert!(is_base_array_tag(tag)); assert!(len <= MAX_ARRAY_LENGTH_FOR_ITERATOR); let skewed_ptr = mem.alloc_words(size_of::() + Words(len)); let ptr: *mut Array = skewed_ptr.get_ptr() as *mut Array; - (*ptr).header.tag = TAG_ARRAY; + (*ptr).header.tag = tag; (*ptr).header.init_forward(skewed_ptr); (*ptr).len = len; diff --git a/rts/motoko-rts/src/persistence.rs b/rts/motoko-rts/src/persistence.rs index 1caa04c9463..5be757cf999 100644 --- a/rts/motoko-rts/src/persistence.rs +++ b/rts/motoko-rts/src/persistence.rs @@ -19,7 +19,7 @@ use crate::{ }, rts_trap_with, stable_mem::read_persistence_version, - types::{Value, TAG_BLOB}, + types::{Value, TAG_BLOB_B}, }; use self::compatibility::TypeDescriptor; @@ -205,7 +205,8 @@ pub unsafe fn register_stable_type( new_candid_data: Value, new_type_offsets: Value, ) { - assert_eq!(new_candid_data.tag(), TAG_BLOB); + assert_eq!(new_candid_data.tag(), TAG_BLOB_B); + assert_eq!(new_type_offsets.tag(), TAG_BLOB_B); let mut new_type = TypeDescriptor::new(new_candid_data, new_type_offsets); let metadata = PersistentMetadata::get(); let old_type = &mut (*metadata).stable_type; diff --git a/rts/motoko-rts/src/persistence/compatibility.rs b/rts/motoko-rts/src/persistence/compatibility.rs index b484021a640..e144a9ff04a 100644 --- a/rts/motoko-rts/src/persistence/compatibility.rs +++ b/rts/motoko-rts/src/persistence/compatibility.rs @@ -8,7 +8,7 @@ use crate::{ constants::WORD_SIZE, idl::TypeVariance, memory::{alloc_blob, Memory}, - types::{Value, Words}, + types::{Value, Words, TAG_BLOB_B}, }; const DEFAULT_VALUE: Value = Value::from_scalar(0); @@ -90,7 +90,7 @@ impl TypeDescriptor { // be used during a single IC message when no GC increment is running in between. pub unsafe fn build_type_table(&mut self, mem: &mut M) -> *mut *mut u8 { let type_count = self.type_count(); - let temporary_blob = alloc_blob(mem, Words(type_count).to_bytes()); + let temporary_blob = alloc_blob(mem, TAG_BLOB_B, Words(type_count).to_bytes()); let offset_table = self.type_offsets.as_blob().payload_const() as *const usize; let type_table = temporary_blob.as_blob_mut().payload_addr() as *mut *mut u8; let candid_data = self.candid_data.as_blob_mut().payload_addr(); @@ -124,7 +124,7 @@ unsafe fn create_type_check_cache( let new_type_count = new_type.type_count(); let words = Words(BitRel::words(old_type_count, new_type_count)); let byte_length = words.to_bytes(); - let blob_value = alloc_blob(mem, byte_length); + let blob_value = alloc_blob(mem, TAG_BLOB_B, byte_length); let ptr = blob_value.as_blob_mut().payload_addr() as *mut usize; let end = blob_value .as_blob() diff --git a/rts/motoko-rts/src/principal_id.rs b/rts/motoko-rts/src/principal_id.rs index d66f4118249..1be577a9ca2 100644 --- a/rts/motoko-rts/src/principal_id.rs +++ b/rts/motoko-rts/src/principal_id.rs @@ -5,7 +5,7 @@ use crate::mem_utils::memcpy_bytes; use crate::memory::{alloc_blob, Memory}; use crate::rts_trap_with; use crate::text::{blob_compare, blob_of_text}; -use crate::types::{Bytes, Value, TAG_BLOB}; +use crate::types::{Bytes, Value, TAG_BLOB_B, TAG_BLOB_T}; use motoko_rts_macros::ic_mem_fn; @@ -13,7 +13,7 @@ use motoko_rts_macros::ic_mem_fn; #[no_mangle] pub unsafe extern "C" fn compute_crc32(blob: Value) -> u32 { - if blob.tag() != TAG_BLOB { + if !blob.is_blob() { panic!("compute_crc32: Blob expected"); } @@ -98,7 +98,7 @@ pub unsafe fn base32_of_checksummed_blob(mem: &mut M, b: Value) -> Va let n = b.as_blob().len(); let mut data = b.as_blob().payload_const(); - let r = alloc_blob(mem, Bytes((n.as_usize() + 4 + 4) / 5 * 8)); // contains padding + let r = alloc_blob(mem, TAG_BLOB_T, Bytes((n.as_usize() + 4 + 4) / 5 * 8)); // contains padding let blob = r.as_blob_mut(); let dest = blob.payload_addr(); @@ -185,7 +185,7 @@ pub unsafe fn base32_to_blob(mem: &mut M, b: Value) -> Value { let mut data = b.as_blob().payload_const(); // Every group of 8 characters will yield 5 bytes - let r = alloc_blob(mem, Bytes(((n.as_usize() + 7) / 8) * 5)); // we deal with padding later + let r = alloc_blob(mem, TAG_BLOB_B, Bytes(((n.as_usize() + 7) / 8) * 5)); // we deal with padding later let blob = r.as_blob_mut(); let dest = blob.payload_addr(); @@ -225,7 +225,7 @@ unsafe fn base32_to_principal(mem: &mut M, b: Value) -> Value { let mut data = blob.payload_const(); // Every group of 5 characters will yield 6 bytes (due to the hypen) - let r = alloc_blob(mem, Bytes(((n.as_usize() + 4) / 5) * 6)); + let r = alloc_blob(mem, TAG_BLOB_T, Bytes(((n.as_usize() + 4) / 5) * 6)); let blob = r.as_blob_mut(); let mut dest = blob.payload_addr(); @@ -269,7 +269,7 @@ pub unsafe fn blob_of_principal(mem: &mut M, t: Value) -> Value { rts_trap_with("blob_of_principal: principal too short"); } - let stripped = alloc_blob(mem, bytes_len - Bytes(4)); + let stripped = alloc_blob(mem, TAG_BLOB_B, bytes_len - Bytes(4)); memcpy_bytes( stripped.as_blob_mut().payload_addr() as usize, bytes.as_blob().payload_const().add(4) as usize, @@ -284,3 +284,15 @@ pub unsafe fn blob_of_principal(mem: &mut M, t: Value) -> Value { allocation_barrier(stripped) } + +// for testing +pub unsafe fn blob_of_ptr_size(mem: &mut M, buf: *const u8, n: Bytes) -> Value { + let blob = alloc_blob(mem, TAG_BLOB_B, n); + let payload_addr = blob.as_blob_mut().payload_addr(); + memcpy_bytes(payload_addr as usize, buf as usize, n); + allocation_barrier(blob) +} + +pub unsafe fn blob_of_str(mem: &mut M, s: &str) -> Value { + blob_of_ptr_size(mem, s.as_ptr(), Bytes(s.len())) +} diff --git a/rts/motoko-rts/src/region.rs b/rts/motoko-rts/src/region.rs index cb26d12962b..cc7a915f2ae 100644 --- a/rts/motoko-rts/src/region.rs +++ b/rts/motoko-rts/src/region.rs @@ -1,7 +1,7 @@ use crate::barriers::{allocation_barrier, init_with_barrier, write_with_barrier}; use crate::memory::{alloc_blob, Memory}; use crate::trap_with_prefix; -use crate::types::{size_of, Blob, Bytes, Region, Value, TAG_REGION}; +use crate::types::{size_of, Blob, Bytes, Region, Value, TAG_BLOB_B, TAG_REGION}; // Versions // Should agree with constants in StableMem in compile.ml @@ -499,7 +499,7 @@ pub unsafe fn region_new(mem: &mut M) -> Value { meta_data::total_allocated_regions::set(next_id + 1); - let vec_pages = alloc_blob(mem, Bytes(0)); + let vec_pages = alloc_blob(mem, TAG_BLOB_B, Bytes(0)); allocation_barrier(vec_pages); let r_ptr = alloc_region(mem, next_id, 0, vec_pages); @@ -535,6 +535,7 @@ pub unsafe fn region_recover(mem: &mut M, rid: &RegionId) -> Value { let block_count = (page_count + PAGES_IN_BLOCK - 1) / PAGES_IN_BLOCK; let vec_pages = alloc_blob( mem, + TAG_BLOB_B, Bytes(block_count as usize * bytes_of::() as usize), ); @@ -679,7 +680,8 @@ pub(crate) unsafe fn region_migration_from_some_stable_memory(mem: &m // Temp for the head block, which we move to be physically last. // NB: no allocation_barrier is required: header_val is temporary and can be reclaimed by the next GC increment/run. // TODO: instead of allocating an 8MB blob, just stack-allocate a tmp page and zero page, and transfer/zero-init via the stack, using a loop. - let header_val = crate::memory::alloc_blob(mem, crate::types::Bytes(header_len as usize)); + let header_val = + crate::memory::alloc_blob(mem, TAG_BLOB_B, crate::types::Bytes(header_len as usize)); let header_blob = header_val.as_blob_mut(); let header_bytes = core::slice::from_raw_parts_mut(header_blob.payload_addr(), header_len as usize); @@ -882,6 +884,7 @@ pub unsafe fn region_grow(mem: &mut M, r: Value, new_pages: u64) -> u let new_vec_pages = alloc_blob( mem, + TAG_BLOB_B, Bytes(new_block_count as usize * meta_data::bytes_of::() as usize), ); let old_vec_byte_count = old_block_count * meta_data::bytes_of::() as u32; @@ -1073,7 +1076,7 @@ pub(crate) unsafe fn region_load_blob( offset: u64, len: usize, ) -> Value { - let blob_val = crate::memory::alloc_blob(mem, crate::types::Bytes(len as usize)); + let blob_val = crate::memory::alloc_blob(mem, TAG_BLOB_B, crate::types::Bytes(len as usize)); let blob = blob_val.as_blob_mut(); if len < (isize::MAX as usize) { diff --git a/rts/motoko-rts/src/stabilization/deserialization.rs b/rts/motoko-rts/src/stabilization/deserialization.rs index f94affdc97f..7a00834e1e8 100644 --- a/rts/motoko-rts/src/stabilization/deserialization.rs +++ b/rts/motoko-rts/src/stabilization/deserialization.rs @@ -3,9 +3,10 @@ pub mod stable_memory_access; use crate::{ constants::MB, + gc::incremental::array_slicing::slice_array, memory::Memory, stabilization::deserialization::scan_stack::STACK_EMPTY, - types::{FwdPtr, Tag, Value, TAG_ARRAY, TAG_ARRAY_SLICE_MIN, TAG_FWD_PTR}, + types::{FwdPtr, Tag, Value, TAG_ARRAY_SLICE_MIN, TAG_FWD_PTR}, visitor::visit_pointer_fields, }; @@ -96,24 +97,16 @@ impl Deserialization { |context, field_address| { *field_address = translate(context, *field_address); }, - |context, slice_start, array| { - const SLICE_INCREMENT: usize = 127; - debug_assert!(SLICE_INCREMENT >= TAG_ARRAY_SLICE_MIN); - if array.len() - slice_start > SLICE_INCREMENT { - let new_start = slice_start + SLICE_INCREMENT; - // Remember to visit the next array slice: - // Update the tag and push the array back on the stack. - (*array).header.tag = new_start; + |context, _, array| { + let length = slice_array(array); + if (*array).header.tag >= TAG_ARRAY_SLICE_MIN { + // Push the array back on the stack to visit the next array slice. context .deserialization .scan_stack .push(context.mem, target_object); - new_start - } else { - // No further visits of this array. Restore the tag. - (*array).header.tag = TAG_ARRAY; - array.len() } + length }, ); } diff --git a/rts/motoko-rts/src/stabilization/deserialization/scan_stack.rs b/rts/motoko-rts/src/stabilization/deserialization/scan_stack.rs index ab175e013c8..d426cc8dd51 100644 --- a/rts/motoko-rts/src/stabilization/deserialization/scan_stack.rs +++ b/rts/motoko-rts/src/stabilization/deserialization/scan_stack.rs @@ -39,7 +39,7 @@ use core::ptr::null_mut; use crate::memory::{alloc_blob, Memory}; -use crate::types::{size_of, Blob, Value}; +use crate::types::{size_of, Blob, Value, TAG_BLOB_B}; pub struct ScanStack { last: *mut StackTable, @@ -106,8 +106,8 @@ impl ScanStack { unsafe fn new_table(mem: &mut M, previous: *mut StackTable) -> *mut StackTable { // No post allocation barrier as this RTS-internal blob will be collected by the GC. - let table = - alloc_blob(mem, size_of::().to_bytes()).as_blob_mut() as *mut StackTable; + let table = alloc_blob(mem, TAG_BLOB_B, size_of::().to_bytes()).as_blob_mut() + as *mut StackTable; (*table).previous = previous; (*table).next = null_mut(); if previous != null_mut() { diff --git a/rts/motoko-rts/src/stabilization/ic/metadata.rs b/rts/motoko-rts/src/stabilization/ic/metadata.rs index bca6d570174..2ddcf99b5ad 100644 --- a/rts/motoko-rts/src/stabilization/ic/metadata.rs +++ b/rts/motoko-rts/src/stabilization/ic/metadata.rs @@ -38,7 +38,7 @@ use crate::{ get_version, ic0_stable64_read, ic0_stable64_size, ic0_stable64_write, read_u32, read_u64, set_version, write_u32, write_u64, PAGE_SIZE, }, - types::{size_of, Bytes, Value}, + types::{size_of, Bytes, Tag, Value, TAG_BLOB_B}, }; use super::performance::InstructionMeter; @@ -112,10 +112,10 @@ impl StabilizationMetadata { length } - fn read_blob(mem: &mut M, offset: &mut u64) -> Value { + fn read_blob(mem: &mut M, tag: Tag, offset: &mut u64) -> Value { let length = Self::read_length(offset); unsafe { - let value = alloc_blob(mem, Bytes(length as usize)); + let value = alloc_blob(mem, tag, Bytes(length as usize)); ic0_stable64_read( value.as_blob_mut().payload_addr() as u64, *offset, @@ -129,8 +129,8 @@ impl StabilizationMetadata { } fn load_type_descriptor(mem: &mut M, offset: &mut u64) -> TypeDescriptor { - let candid_data = Self::read_blob(mem, offset); - let type_offsets = Self::read_blob(mem, offset); + let candid_data = Self::read_blob(mem, TAG_BLOB_B, offset); + let type_offsets = Self::read_blob(mem, TAG_BLOB_B, offset); TypeDescriptor::new(candid_data, type_offsets) } diff --git a/rts/motoko-rts/src/stabilization/layout.rs b/rts/motoko-rts/src/stabilization/layout.rs index a090af84ed9..117850a3807 100644 --- a/rts/motoko-rts/src/stabilization/layout.rs +++ b/rts/motoko-rts/src/stabilization/layout.rs @@ -29,9 +29,10 @@ use crate::{ memory::Memory, rts_trap_with, types::{ - size_of, Tag, Value, TAG_ARRAY, TAG_ARRAY_SLICE_MIN, TAG_BIGINT, TAG_BITS64, TAG_BLOB, - TAG_CONCAT, TAG_MUTBOX, TAG_OBJECT, TAG_OBJ_IND, TAG_REGION, TAG_SOME, TAG_VARIANT, - TRUE_VALUE, + base_array_tag, size_of, Tag, Value, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S, + TAG_ARRAY_SLICE_MIN, TAG_ARRAY_T, TAG_BIGINT, TAG_BITS64_F, TAG_BITS64_S, TAG_BITS64_U, + TAG_BLOB_A, TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T, TAG_CONCAT, TAG_MUTBOX, TAG_OBJECT, + TAG_OBJ_IND, TAG_REGION, TAG_SOME, TAG_VARIANT, TRUE_VALUE, }, }; @@ -64,19 +65,27 @@ mod stable_variant; /// Different kinds of objects used in the stable format. #[repr(u64)] -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, Copy, PartialEq, Debug)] pub enum StableObjectKind { - Array = 1, - MutBox = 2, - Object = 3, - Blob = 4, - Bits64 = 5, - Region = 6, - Variant = 7, - Concat = 8, - BigInt = 9, - ObjInd = 10, - Some = 11, + ArrayImmutable = 1, + ArrayMutable = 2, + ArrayTuple = 3, + ArraySharedFunction = 4, + MutBox = 5, + Object = 6, + BlobBytes = 7, + BlobText = 8, + BlobPrincipal = 9, + BlobActor = 10, + Bits64Unsigned = 11, + Bits64Signed = 12, + Bits64Float = 13, + Region = 14, + Variant = 15, + Concat = 16, + BigInt = 17, + ObjInd = 18, + Some = 19, } #[repr(C)] @@ -90,11 +99,19 @@ impl StableObjectKind { impl StableTag { pub fn decode(&self) -> StableObjectKind { - const STABLE_TAG_ARRAY: u64 = StableObjectKind::Array as u64; + const STABLE_TAG_ARRAY_IMMUTABLE: u64 = StableObjectKind::ArrayImmutable as u64; + const STABLE_TAG_ARRAY_MUTABLE: u64 = StableObjectKind::ArrayMutable as u64; + const STABLE_TAG_ARRAY_TUPLE: u64 = StableObjectKind::ArrayTuple as u64; + const STABLE_TAG_ARRAY_SHARED_FUNCTION: u64 = StableObjectKind::ArraySharedFunction as u64; const STABLE_TAG_MUTBOX: u64 = StableObjectKind::MutBox as u64; const STABLE_TAG_OBJECT: u64 = StableObjectKind::Object as u64; - const STABLE_TAG_BLOB: u64 = StableObjectKind::Blob as u64; - const STABLE_TAG_BITS64: u64 = StableObjectKind::Bits64 as u64; + const STABLE_TAG_BLOB_BYTES: u64 = StableObjectKind::BlobBytes as u64; + const STABLE_TAG_BLOB_TEXT: u64 = StableObjectKind::BlobText as u64; + const STABLE_TAG_BLOB_PRINCIPAL: u64 = StableObjectKind::BlobPrincipal as u64; + const STABLE_TAG_BLOB_ACTOR: u64 = StableObjectKind::BlobActor as u64; + const STABLE_TAG_BITS64_UNSIGNED: u64 = StableObjectKind::Bits64Unsigned as u64; + const STABLE_TAG_BITS64_SIGNED: u64 = StableObjectKind::Bits64Signed as u64; + const STABLE_TAG_BITS64_FLOAT: u64 = StableObjectKind::Bits64Float as u64; const STABLE_TAG_REGION: u64 = StableObjectKind::Region as u64; const STABLE_TAG_VARIANT: u64 = StableObjectKind::Variant as u64; const STABLE_TAG_CONCAT: u64 = StableObjectKind::Concat as u64; @@ -102,11 +119,19 @@ impl StableTag { const STABLE_TAG_OBJIND: u64 = StableObjectKind::ObjInd as u64; const STABLE_TAG_SOME: u64 = StableObjectKind::Some as u64; match self.0 { - STABLE_TAG_ARRAY => StableObjectKind::Array, + STABLE_TAG_ARRAY_IMMUTABLE => StableObjectKind::ArrayImmutable, + STABLE_TAG_ARRAY_MUTABLE => StableObjectKind::ArrayMutable, + STABLE_TAG_ARRAY_TUPLE => StableObjectKind::ArrayTuple, + STABLE_TAG_ARRAY_SHARED_FUNCTION => StableObjectKind::ArraySharedFunction, STABLE_TAG_MUTBOX => StableObjectKind::MutBox, STABLE_TAG_OBJECT => StableObjectKind::Object, - STABLE_TAG_BLOB => StableObjectKind::Blob, - STABLE_TAG_BITS64 => StableObjectKind::Bits64, + STABLE_TAG_BLOB_BYTES => StableObjectKind::BlobBytes, + STABLE_TAG_BLOB_TEXT => StableObjectKind::BlobText, + STABLE_TAG_BLOB_PRINCIPAL => StableObjectKind::BlobPrincipal, + STABLE_TAG_BLOB_ACTOR => StableObjectKind::BlobActor, + STABLE_TAG_BITS64_UNSIGNED => StableObjectKind::Bits64Unsigned, + STABLE_TAG_BITS64_SIGNED => StableObjectKind::Bits64Signed, + STABLE_TAG_BITS64_FLOAT => StableObjectKind::Bits64Float, STABLE_TAG_REGION => StableObjectKind::Region, STABLE_TAG_VARIANT => StableObjectKind::Variant, STABLE_TAG_CONCAT => StableObjectKind::Concat, @@ -123,11 +148,24 @@ impl StableObjectKind { match tag { // During the marking phase of the incremental GC, the mutator can see // array slice information in the object tag. - TAG_ARRAY | TAG_ARRAY_SLICE_MIN.. => StableObjectKind::Array, + TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S | TAG_ARRAY_SLICE_MIN.. => { + match base_array_tag(tag) { + TAG_ARRAY_I => StableObjectKind::ArrayImmutable, + TAG_ARRAY_M => StableObjectKind::ArrayMutable, + TAG_ARRAY_T => StableObjectKind::ArrayTuple, + TAG_ARRAY_S => StableObjectKind::ArraySharedFunction, + _ => unreachable!("invalid array tag"), + } + } TAG_MUTBOX => StableObjectKind::MutBox, TAG_OBJECT => StableObjectKind::Object, - TAG_BLOB => StableObjectKind::Blob, - TAG_BITS64 => StableObjectKind::Bits64, + TAG_BLOB_B => StableObjectKind::BlobBytes, + TAG_BLOB_T => StableObjectKind::BlobText, + TAG_BLOB_P => StableObjectKind::BlobPrincipal, + TAG_BLOB_A => StableObjectKind::BlobActor, + TAG_BITS64_U => StableObjectKind::Bits64Unsigned, + TAG_BITS64_S => StableObjectKind::Bits64Signed, + TAG_BITS64_F => StableObjectKind::Bits64Float, TAG_REGION => StableObjectKind::Region, TAG_VARIANT => StableObjectKind::Variant, TAG_CONCAT => StableObjectKind::Concat, @@ -251,11 +289,15 @@ where ) { } - unsafe fn allocate_deserialized(&self, main_memory: &mut M) -> Value { + unsafe fn allocate_deserialized( + &self, + main_memory: &mut M, + _object_kind: StableObjectKind, + ) -> Value { main_memory.alloc_words(size_of::()) } - unsafe fn deserialize_static_part(&self, target_object: *mut T); + unsafe fn deserialize_static_part(&self, target_object: *mut T, object_kind: StableObjectKind); unsafe fn deserialize_dynamic_part( &self, @@ -270,12 +312,13 @@ where main_memory: &mut M, stable_memory: &StableMemoryAccess, stable_object: StableValue, + object_kind: StableObjectKind, ) -> Value { let stable_address = stable_object.payload_address(); let stable_static_part = stable_memory.read::(stable_address); - let target = stable_static_part.allocate_deserialized(main_memory); + let target = stable_static_part.allocate_deserialized(main_memory, object_kind); let target_object = target.get_ptr() as *mut T; - stable_static_part.deserialize_static_part(target_object); + stable_static_part.deserialize_static_part(target_object, object_kind); stable_static_part.deserialize_dynamic_part( main_memory, stable_memory, @@ -313,11 +356,19 @@ pub fn scan_serialized< } let tag = context.serialization.to_space().read::(); match tag.decode() { - StableObjectKind::Array => StableArray::scan_serialized(context, translate), + StableObjectKind::ArrayImmutable + | StableObjectKind::ArrayMutable + | StableObjectKind::ArrayTuple + | StableObjectKind::ArraySharedFunction => StableArray::scan_serialized(context, translate), StableObjectKind::MutBox => StableMutBox::scan_serialized(context, translate), StableObjectKind::Object => StableObject::scan_serialized(context, translate), - StableObjectKind::Blob => StableBlob::scan_serialized(context, translate), - StableObjectKind::Bits64 => StableBits64::scan_serialized(context, translate), + StableObjectKind::BlobBytes + | StableObjectKind::BlobText + | StableObjectKind::BlobPrincipal + | StableObjectKind::BlobActor => StableBlob::scan_serialized(context, translate), + StableObjectKind::Bits64Unsigned + | StableObjectKind::Bits64Signed + | StableObjectKind::Bits64Float => StableBits64::scan_serialized(context, translate), StableObjectKind::Region => StableRegion::scan_serialized(context, translate), StableObjectKind::Variant => StableVariant::scan_serialized(context, translate), StableObjectKind::Concat => StableConcat::scan_serialized(context, translate), @@ -329,11 +380,21 @@ pub fn scan_serialized< pub unsafe fn serialize(stable_memory: &mut StableMemoryStream, main_object: Value) { match StableObjectKind::deserialize(main_object.tag()) { - StableObjectKind::Array => StableArray::serialize(stable_memory, main_object), + StableObjectKind::ArrayImmutable + | StableObjectKind::ArrayMutable + | StableObjectKind::ArrayTuple + | StableObjectKind::ArraySharedFunction => { + StableArray::serialize(stable_memory, main_object) + } StableObjectKind::MutBox => StableMutBox::serialize(stable_memory, main_object), StableObjectKind::Object => StableObject::serialize(stable_memory, main_object), - StableObjectKind::Blob => StableBlob::serialize(stable_memory, main_object), - StableObjectKind::Bits64 => StableBits64::serialize(stable_memory, main_object), + StableObjectKind::BlobBytes + | StableObjectKind::BlobText + | StableObjectKind::BlobPrincipal + | StableObjectKind::BlobActor => StableBlob::serialize(stable_memory, main_object), + StableObjectKind::Bits64Unsigned + | StableObjectKind::Bits64Signed + | StableObjectKind::Bits64Float => StableBits64::serialize(stable_memory, main_object), StableObjectKind::Region => StableRegion::serialize(stable_memory, main_object), StableObjectKind::Variant => StableVariant::serialize(stable_memory, main_object), StableObjectKind::Concat => StableConcat::serialize(stable_memory, main_object), @@ -349,39 +410,48 @@ pub unsafe fn deserialize( stable_object: StableValue, ) -> Value { let tag = stable_memory.read::(stable_object.to_stable_address()); - match tag.decode() { - StableObjectKind::Array => { - StableArray::deserialize(main_memory, stable_memory, stable_object) + let object_kind = tag.decode(); + match object_kind { + StableObjectKind::ArrayImmutable + | StableObjectKind::ArrayMutable + | StableObjectKind::ArrayTuple + | StableObjectKind::ArraySharedFunction => { + StableArray::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::MutBox => { - StableMutBox::deserialize(main_memory, stable_memory, stable_object) + StableMutBox::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::Object => { - StableObject::deserialize(main_memory, stable_memory, stable_object) + StableObject::deserialize(main_memory, stable_memory, stable_object, object_kind) } - StableObjectKind::Blob => { - StableBlob::deserialize(main_memory, stable_memory, stable_object) + StableObjectKind::BlobBytes + | StableObjectKind::BlobText + | StableObjectKind::BlobPrincipal + | StableObjectKind::BlobActor => { + StableBlob::deserialize(main_memory, stable_memory, stable_object, object_kind) } - StableObjectKind::Bits64 => { - StableBits64::deserialize(main_memory, stable_memory, stable_object) + StableObjectKind::Bits64Unsigned + | StableObjectKind::Bits64Signed + | StableObjectKind::Bits64Float => { + StableBits64::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::Region => { - StableRegion::deserialize(main_memory, stable_memory, stable_object) + StableRegion::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::Variant => { - StableVariant::deserialize(main_memory, stable_memory, stable_object) + StableVariant::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::Concat => { - StableConcat::deserialize(main_memory, stable_memory, stable_object) + StableConcat::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::BigInt => { - StableBigInt::deserialize(main_memory, stable_memory, stable_object) + StableBigInt::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::ObjInd => { - StableObjInd::deserialize(main_memory, stable_memory, stable_object) + StableObjInd::deserialize(main_memory, stable_memory, stable_object, object_kind) } StableObjectKind::Some => { - StableSome::deserialize(main_memory, stable_memory, stable_object) + StableSome::deserialize(main_memory, stable_memory, stable_object, object_kind) } } } diff --git a/rts/motoko-rts/src/stabilization/layout/stable_array.rs b/rts/motoko-rts/src/stabilization/layout/stable_array.rs index d363d4f5fce..7059e55ca38 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_array.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_array.rs @@ -8,10 +8,10 @@ use crate::{ ArraySlice, SerializationContext, }, }, - types::{size_of, Array, Value, TAG_ARRAY}, + types::{size_of, Array, Tag, Value, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_S, TAG_ARRAY_T}, }; -use super::{Serializer, StableToSpace, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableToSpace, StableValue, StaticScanner}; #[repr(C)] pub struct StableArray { @@ -60,13 +60,22 @@ impl Serializer for StableArray { } } - unsafe fn allocate_deserialized(&self, main_memory: &mut M) -> Value { + unsafe fn allocate_deserialized( + &self, + main_memory: &mut M, + object_kind: StableObjectKind, + ) -> Value { + let array_tag = decode_array_tag(object_kind); let array_length = self.array_length as usize; - alloc_array(main_memory, array_length) + alloc_array(main_memory, array_tag, array_length) } - unsafe fn deserialize_static_part(&self, target_array: *mut Array) { - debug_assert_eq!((*target_array).header.tag, TAG_ARRAY); + unsafe fn deserialize_static_part( + &self, + target_array: *mut Array, + object_kind: StableObjectKind, + ) { + debug_assert_eq!((*target_array).header.tag, decode_array_tag(object_kind)); debug_assert_eq!((*target_array).len as u64, self.array_length); } @@ -88,6 +97,16 @@ impl Serializer for StableArray { } } +fn decode_array_tag(object_kind: StableObjectKind) -> Tag { + match object_kind { + StableObjectKind::ArrayImmutable => TAG_ARRAY_I, + StableObjectKind::ArrayMutable => TAG_ARRAY_M, + StableObjectKind::ArrayTuple => TAG_ARRAY_T, + StableObjectKind::ArraySharedFunction => TAG_ARRAY_S, + _ => unreachable!("invalid stable array tag"), + } +} + impl StableArray { pub fn resume_scanning< 'a, diff --git a/rts/motoko-rts/src/stabilization/layout/stable_bigint.rs b/rts/motoko-rts/src/stabilization/layout/stable_bigint.rs index e543de9d4ea..2d4d9da3c25 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_bigint.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_bigint.rs @@ -8,7 +8,9 @@ use crate::stabilization::serialization::SerializationContext; use crate::tommath_bindings::mp_digit; use crate::types::{size_of, BigInt, Bytes, Value, TAG_BIGINT}; -use super::{round_to_u64, Serializer, StableToSpace, StableValue, StaticScanner}; +use super::{ + round_to_u64, Serializer, StableObjectKind, StableToSpace, StableValue, StaticScanner, +}; // Tom's math library, as configured for Motoko RTS, encodes a big numbers as an array of 64-bit // elements, where each element stores 60 bits of the number while its highest 4 bits are zero. @@ -137,14 +139,24 @@ impl Serializer for StableBigInt { .skip(rounded_length as usize); } - unsafe fn allocate_deserialized(&self, main_memory: &mut M) -> Value { + unsafe fn allocate_deserialized( + &self, + main_memory: &mut M, + object_kind: StableObjectKind, + ) -> Value { + debug_assert_eq!(object_kind, StableObjectKind::BigInt); let elements = self.deserialized_elements(); let payload = mp_calloc(main_memory, elements, Bytes(ELEMENT_SIZE)) as *mut mp_digit; let bigint = BigInt::from_payload(payload); Value::from_ptr(bigint as usize) } - unsafe fn deserialize_static_part(&self, target_bigint: *mut BigInt) { + unsafe fn deserialize_static_part( + &self, + target_bigint: *mut BigInt, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::BigInt); debug_assert_eq!((*target_bigint).header.tag, TAG_BIGINT); let elements = self.deserialized_elements(); debug_assert_eq!((*target_bigint).mp_int.alloc as usize, elements); diff --git a/rts/motoko-rts/src/stabilization/layout/stable_bits64.rs b/rts/motoko-rts/src/stabilization/layout/stable_bits64.rs index f3d48229236..a24a3dbb20d 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_bits64.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_bits64.rs @@ -1,9 +1,9 @@ use crate::{ stabilization::serialization::stable_memory_stream::StableMemoryStream, - types::{Bits64, Value, TAG_BITS64}, + types::{Bits64, Tag, Value, TAG_BITS64_F, TAG_BITS64_S, TAG_BITS64_U}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableBits64 { @@ -22,11 +22,24 @@ impl Serializer for StableBits64 { } } - unsafe fn deserialize_static_part(&self, target_bits64: *mut Bits64) { - (*target_bits64).header.tag = TAG_BITS64; + unsafe fn deserialize_static_part( + &self, + target_bits64: *mut Bits64, + object_kind: StableObjectKind, + ) { + (*target_bits64).header.tag = decode_bits64_tag(object_kind); (*target_bits64) .header .init_forward(Value::from_ptr(target_bits64 as usize)); (*target_bits64).bits = self.bits as usize; } } + +fn decode_bits64_tag(object_kind: StableObjectKind) -> Tag { + match object_kind { + StableObjectKind::Bits64Unsigned => TAG_BITS64_U, + StableObjectKind::Bits64Signed => TAG_BITS64_S, + StableObjectKind::Bits64Float => TAG_BITS64_F, + _ => unreachable!("invalid stable boxed word 64 tag"), + } +} diff --git a/rts/motoko-rts/src/stabilization/layout/stable_blob.rs b/rts/motoko-rts/src/stabilization/layout/stable_blob.rs index b7e5cf5e83d..97620328e9e 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_blob.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_blob.rs @@ -7,11 +7,12 @@ use crate::{ SerializationContext, }, }, - types::{size_of, Blob, Bytes, Value, TAG_BLOB}, + types::{size_of, Blob, Bytes, Tag, Value, TAG_BLOB_A, TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T}, }; use super::{ - round_to_u64, write_padding_u64, Serializer, StableToSpace, StableValue, StaticScanner, + round_to_u64, write_padding_u64, Serializer, StableObjectKind, StableToSpace, StableValue, + StaticScanner, }; #[repr(C)] @@ -66,13 +67,22 @@ impl Serializer for StableBlob { .skip(rounded_length as usize); } - unsafe fn allocate_deserialized(&self, main_memory: &mut M) -> Value { + unsafe fn allocate_deserialized( + &self, + main_memory: &mut M, + object_kind: StableObjectKind, + ) -> Value { + let blob_tag = decode_blob_tag(object_kind); let blob_length = self.byte_length as usize; - alloc_blob(main_memory, Bytes(blob_length)) + alloc_blob(main_memory, blob_tag, Bytes(blob_length)) } - unsafe fn deserialize_static_part(&self, target_blob: *mut Blob) { - debug_assert_eq!((*target_blob).header.tag, TAG_BLOB); + unsafe fn deserialize_static_part( + &self, + target_blob: *mut Blob, + object_kind: StableObjectKind, + ) { + debug_assert_eq!((*target_blob).header.tag, decode_blob_tag(object_kind)); debug_assert_eq!((*target_blob).len.as_usize(), self.byte_length as usize); } @@ -90,3 +100,13 @@ impl Serializer for StableBlob { stable_memory.raw_read(source_payload, target_payload, blob_length); } } + +fn decode_blob_tag(object_kind: StableObjectKind) -> Tag { + match object_kind { + StableObjectKind::BlobBytes => TAG_BLOB_B, + StableObjectKind::BlobText => TAG_BLOB_T, + StableObjectKind::BlobPrincipal => TAG_BLOB_P, + StableObjectKind::BlobActor => TAG_BLOB_A, + _ => unreachable!("invalid stable blob tag"), + } +} diff --git a/rts/motoko-rts/src/stabilization/layout/stable_concat.rs b/rts/motoko-rts/src/stabilization/layout/stable_concat.rs index 815955d947f..1f31c02c209 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_concat.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_concat.rs @@ -3,7 +3,7 @@ use crate::{ types::{Bytes, Concat, Value, TAG_CONCAT}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableConcat { @@ -36,7 +36,12 @@ impl Serializer for StableConcat { } } - unsafe fn deserialize_static_part(&self, target_concat: *mut Concat) { + unsafe fn deserialize_static_part( + &self, + target_concat: *mut Concat, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::Concat); let n_bytes = Bytes(self.number_of_bytes as usize); (*target_concat).header.tag = TAG_CONCAT; (*target_concat) diff --git a/rts/motoko-rts/src/stabilization/layout/stable_mutbox.rs b/rts/motoko-rts/src/stabilization/layout/stable_mutbox.rs index 9ca6c61e5c0..e9b1f8f4160 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_mutbox.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_mutbox.rs @@ -3,7 +3,7 @@ use crate::{ types::{MutBox, Value, TAG_MUTBOX}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableMutBox { @@ -31,7 +31,12 @@ impl Serializer for StableMutBox { } } - unsafe fn deserialize_static_part(&self, target_mutbox: *mut MutBox) { + unsafe fn deserialize_static_part( + &self, + target_mutbox: *mut MutBox, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::MutBox); (*target_mutbox).header.tag = TAG_MUTBOX; (*target_mutbox) .header diff --git a/rts/motoko-rts/src/stabilization/layout/stable_obj_ind.rs b/rts/motoko-rts/src/stabilization/layout/stable_obj_ind.rs index c2531d3d46a..60697fb2284 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_obj_ind.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_obj_ind.rs @@ -3,7 +3,7 @@ use crate::{ types::{ObjInd, Value, TAG_OBJ_IND}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableObjInd { @@ -31,7 +31,12 @@ impl Serializer for StableObjInd { } } - unsafe fn deserialize_static_part(&self, target_obj_ind: *mut ObjInd) { + unsafe fn deserialize_static_part( + &self, + target_obj_ind: *mut ObjInd, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::ObjInd); (*target_obj_ind).header.tag = TAG_OBJ_IND; (*target_obj_ind) .header diff --git a/rts/motoko-rts/src/stabilization/layout/stable_object.rs b/rts/motoko-rts/src/stabilization/layout/stable_object.rs index e6fb818eddf..94ec0c41910 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_object.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_object.rs @@ -72,12 +72,22 @@ impl Serializer for StableObject { } } - unsafe fn allocate_deserialized(&self, main_memory: &mut M) -> Value { + unsafe fn allocate_deserialized( + &self, + main_memory: &mut M, + object_kind: StableObjectKind, + ) -> Value { + debug_assert_eq!(object_kind, StableObjectKind::Object); let total_size = size_of::() + Words(self.size as usize); main_memory.alloc_words(total_size) } - unsafe fn deserialize_static_part(&self, target_object: *mut Object) { + unsafe fn deserialize_static_part( + &self, + target_object: *mut Object, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::Object); (*target_object).header.tag = TAG_OBJECT; (*target_object) .header @@ -123,7 +133,7 @@ fn get_object_size(stable_memory: &StableMemoryStream, main_object: *mut Object) let target_location = (*(main_hash_blob.get_ptr() as *mut FwdPtr)).fwd; let stable_offset = target_location.get_ptr() as u64; let stable_hash_blob = stable_memory.read_preceding::(stable_offset); - assert!(stable_hash_blob.tag.decode() == StableObjectKind::Blob); + assert!(stable_hash_blob.tag.decode() == StableObjectKind::BlobBytes); let hash_blob_length = stable_hash_blob.header.byte_length() as usize; let hash_entry_length = size_of::().to_bytes().as_usize(); debug_assert_eq!(hash_blob_length % hash_entry_length, 0); diff --git a/rts/motoko-rts/src/stabilization/layout/stable_region.rs b/rts/motoko-rts/src/stabilization/layout/stable_region.rs index 7bbe9cedcfd..c154dca99f0 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_region.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_region.rs @@ -3,7 +3,7 @@ use crate::{ types::{Region, Value, TAG_REGION}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableRegion { @@ -35,7 +35,12 @@ impl Serializer for StableRegion { } } - unsafe fn deserialize_static_part(&self, target_region: *mut Region) { + unsafe fn deserialize_static_part( + &self, + target_region: *mut Region, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::Region); (*target_region).header.tag = TAG_REGION; (*target_region) .header diff --git a/rts/motoko-rts/src/stabilization/layout/stable_some.rs b/rts/motoko-rts/src/stabilization/layout/stable_some.rs index 3b0d665c85a..29b118037f7 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_some.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_some.rs @@ -3,7 +3,7 @@ use crate::{ types::{Some, Value, TAG_SOME}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableSome { @@ -31,7 +31,12 @@ impl Serializer for StableSome { } } - unsafe fn deserialize_static_part(&self, target_some: *mut Some) { + unsafe fn deserialize_static_part( + &self, + target_some: *mut Some, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::Some); (*target_some).header.tag = TAG_SOME; (*target_some) .header diff --git a/rts/motoko-rts/src/stabilization/layout/stable_variant.rs b/rts/motoko-rts/src/stabilization/layout/stable_variant.rs index 54caa9798c7..1016d00e75c 100644 --- a/rts/motoko-rts/src/stabilization/layout/stable_variant.rs +++ b/rts/motoko-rts/src/stabilization/layout/stable_variant.rs @@ -3,7 +3,7 @@ use crate::{ types::{Value, Variant, TAG_VARIANT}, }; -use super::{Serializer, StableValue, StaticScanner}; +use super::{Serializer, StableObjectKind, StableValue, StaticScanner}; #[repr(C)] pub struct StableVariant { @@ -33,7 +33,12 @@ impl Serializer for StableVariant { } } - unsafe fn deserialize_static_part(&self, target_variant: *mut Variant) { + unsafe fn deserialize_static_part( + &self, + target_variant: *mut Variant, + object_kind: StableObjectKind, + ) { + debug_assert_eq!(object_kind, StableObjectKind::Variant); (*target_variant).header.tag = TAG_VARIANT; (*target_variant) .header diff --git a/rts/motoko-rts/src/static_checks.rs b/rts/motoko-rts/src/static_checks.rs index cd0a032917a..645aca610e4 100644 --- a/rts/motoko-rts/src/static_checks.rs +++ b/rts/motoko-rts/src/static_checks.rs @@ -1,5 +1,6 @@ //! Compile-time assertions to make sure object layouts are as expected +use crate::constants::*; use crate::types::*; use core::mem::{align_of, size_of}; @@ -53,3 +54,11 @@ const _: () = assert!(align_of::() == WORD_SIZE); const _: () = assert!(align_of::() == WORD_SIZE); const _: () = assert!(align_of::() == WORD_SIZE); const _: () = assert!(align_of::() == WORD_SIZE); + +// Array slicing +// TAG_ARRAY_I is smallest tag +const _: () = + assert!(TAG_ARRAY_I < TAG_ARRAY_M && TAG_ARRAY_M < TAG_ARRAY_T && TAG_ARRAY_T < TAG_ARRAY_S); +// 2-bits suffice to encode base array tag in slice, remaining bits suffice to encode slice start. +const _: () = assert!((TAG_ARRAY_S - TAG_ARRAY_I) / 2 < 4); +const _: () = assert!(MAX_ARRAY_LENGTH_FOR_ITERATOR < (1 << (usize::BITS - 2))); diff --git a/rts/motoko-rts/src/text.rs b/rts/motoko-rts/src/text.rs index 1a7dc99fb5d..eaf4828b77b 100644 --- a/rts/motoko-rts/src/text.rs +++ b/rts/motoko-rts/src/text.rs @@ -31,7 +31,7 @@ use crate::libc_declarations::memcmp; use crate::mem_utils::memcpy_bytes; use crate::memory::{alloc_blob, Memory}; use crate::rts_trap_with; -use crate::types::{size_of, Blob, Bytes, Concat, Value, TAG_BLOB, TAG_CONCAT}; +use crate::types::{size_of, Blob, Bytes, Concat, Value, TAG_BLOB_T, TAG_CONCAT}; use alloc::string::String; use core::cmp::{min, Ordering}; @@ -50,7 +50,7 @@ unsafe fn alloc_text_blob(mem: &mut M, size: Bytes) -> Value { if size > MAX_STR_SIZE { rts_trap_with("alloc_text_blob: Text too large"); } - alloc_blob(mem, size) + alloc_blob(mem, TAG_BLOB_T, size) } #[ic_mem_fn] @@ -134,7 +134,7 @@ unsafe extern "C" fn text_to_buf(mut s: Value, mut buf: *mut u8) { loop { let s_ptr = s.as_obj(); - if s_ptr.tag() == TAG_BLOB { + if s_ptr.tag() == TAG_BLOB_T { let blob = s_ptr.as_blob(); memcpy_bytes(buf as usize, blob.payload_addr() as usize, blob.len()); @@ -173,7 +173,7 @@ unsafe extern "C" fn text_to_buf(mut s: Value, mut buf: *mut u8) { #[ic_mem_fn] pub unsafe fn blob_of_text(mem: &mut M, s: Value) -> Value { let obj = s.as_obj(); - if obj.tag() == TAG_BLOB { + if obj.tag() == TAG_BLOB_T { s } else { let concat = obj.as_concat(); @@ -239,8 +239,8 @@ unsafe fn text_compare_range( ), } } else { - debug_assert_eq!(s1_obj.tag(), TAG_BLOB); - debug_assert_eq!(s2_obj.tag(), TAG_BLOB); + debug_assert_eq!(s1_obj.tag(), TAG_BLOB_T); + debug_assert_eq!(s2_obj.tag(), TAG_BLOB_T); let s1_blob = s1_obj.as_blob(); let s2_blob = s2_obj.as_blob(); @@ -290,7 +290,7 @@ unsafe fn text_get_range( continue; } } else { - debug_assert_eq!(s_obj.tag(), TAG_BLOB); + debug_assert_eq!(s_obj.tag(), TAG_BLOB_T); } break; @@ -320,7 +320,8 @@ pub unsafe extern "C" fn text_compare(s1: Value, s2: Value) -> isize { } } -pub(crate) unsafe fn blob_compare(s1: Value, s2: Value) -> isize { +#[no_mangle] +pub unsafe extern "C" fn blob_compare(s1: Value, s2: Value) -> isize { let n1 = text_size(s1); let n2 = text_size(s2); let n = min(n1, n2); @@ -345,7 +346,7 @@ pub(crate) unsafe fn blob_compare(s1: Value, s2: Value) -> isize { /// Length in characters #[no_mangle] pub unsafe extern "C" fn text_len(text: Value) -> usize { - if text.tag() == TAG_BLOB { + if text.tag() == TAG_BLOB_T { let blob = text.as_blob(); let payload_addr = blob.payload_const(); let len = blob.len(); @@ -429,7 +430,7 @@ where )); let string = to_string(&str); let bytes = string.as_bytes(); - let lowercase = alloc_blob(mem, Bytes(bytes.len())); + let lowercase = alloc_blob(mem, TAG_BLOB_T, Bytes(bytes.len())); let mut i = 0; let target_ptr = lowercase.as_blob_mut().payload_addr(); for b in bytes { diff --git a/rts/motoko-rts/src/text_iter.rs b/rts/motoko-rts/src/text_iter.rs index d975f2750ca..d7b4acdab8d 100644 --- a/rts/motoko-rts/src/text_iter.rs +++ b/rts/motoko-rts/src/text_iter.rs @@ -14,7 +14,7 @@ use crate::barriers::allocation_barrier; use crate::memory::{alloc_array, Memory}; use crate::rts_trap_with; use crate::text::decode_code_point; -use crate::types::{Value, TAG_BLOB, TAG_CONCAT}; +use crate::types::{Value, TAG_ARRAY_T, TAG_BLOB_T, TAG_CONCAT}; use motoko_rts_macros::ic_mem_fn; @@ -28,7 +28,7 @@ unsafe fn find_leaf(mem: &mut M, mut text: Value, todo: *mut Value) - let concat = text.as_concat(); // Add right node to TODOs - let new_todo = alloc_array(mem, 2); + let new_todo = alloc_array(mem, TAG_ARRAY_T, 2); let new_todo_array = new_todo.as_array(); // No pre-update barrier for object initialization, but do perform post-update barrier. new_todo_array.initialize(TODO_TEXT_IDX, (*concat).text2, mem); @@ -40,7 +40,7 @@ unsafe fn find_leaf(mem: &mut M, mut text: Value, todo: *mut Value) - text = (*concat).text1; } - debug_assert_eq!(text.tag(), TAG_BLOB); + debug_assert_eq!(text.tag(), TAG_BLOB_T); text } @@ -55,7 +55,7 @@ const NO_OBJECT: Value = Value::from_scalar(0); /// Returns a new iterator for the text #[ic_mem_fn] pub unsafe fn text_iter(mem: &mut M, text: Value) -> Value { - let iter = alloc_array(mem, 3); + let iter = alloc_array(mem, TAG_ARRAY_T, 3); let array = iter.as_array(); // Initialize the TODO field first, to be able to use it use the location to `find_leaf` @@ -128,7 +128,7 @@ pub unsafe fn text_iter_next(mem: &mut M, iter: Value) -> u32 { text_iter_next(mem, iter) } else { // Otherwise remove the entry from the chain - debug_assert_eq!(text.tag(), TAG_BLOB); + debug_assert_eq!(text.tag(), TAG_BLOB_T); iter_array.set(ITER_BLOB_IDX, text, mem); iter_array.set(ITER_POS_IDX, Value::from_scalar(0), mem); diff --git a/rts/motoko-rts/src/types.rs b/rts/motoko-rts/src/types.rs index 7e23f4a64a4..5cb084d586a 100644 --- a/rts/motoko-rts/src/types.rs +++ b/rts/motoko-rts/src/types.rs @@ -25,7 +25,7 @@ use crate::tommath_bindings::{mp_digit, mp_int}; use core::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign}; use core::ptr::null; -use crate::constants::WORD_SIZE; +use crate::constants::{MAX_ARRAY_LENGTH_FOR_ITERATOR, WORD_SIZE}; use crate::rts_trap_with; pub fn size_of() -> Words { @@ -311,6 +311,16 @@ impl Value { tag != TAG_FWD_PTR && tag != TAG_ONE_WORD_FILLER && tag != TAG_FREE_SPACE } + pub unsafe fn is_blob(self) -> bool { + let tag = self.tag(); + is_blob_tag(tag) + } + + pub unsafe fn is_array(self) -> bool { + let tag = self.tag(); + is_array_or_slice_tag(tag) + } + /// Get the pointer as `Obj` using forwarding. In debug mode panics if the value is not a pointer. pub unsafe fn as_obj(self) -> *mut Obj { debug_assert!(self.get().is_ptr()); @@ -328,7 +338,7 @@ impl Value { /// Get the pointer as `Array` using forwarding. In debug mode panics if the value is not a pointer or the /// pointed object is not an `Array`. pub unsafe fn as_array(self) -> *mut Array { - debug_assert!(self.tag() == TAG_ARRAY || self.tag() >= TAG_ARRAY_SLICE_MIN); + debug_assert!(self.is_array()); self.check_forwarding_pointer(); self.forward().get_ptr() as *mut Array } @@ -365,14 +375,14 @@ impl Value { /// Get the pointer as `Blob` using forwarding. In debug mode panics if the value is not a pointer or the /// pointed object is not a `Blob`. pub unsafe fn as_blob(self) -> *const Blob { - debug_assert_eq!(self.tag(), TAG_BLOB); + debug_assert!(self.is_blob()); self.check_forwarding_pointer(); self.forward().get_ptr() as *const Blob } /// Get the pointer as mutable `Blob` using forwarding. pub unsafe fn as_blob_mut(self) -> *mut Blob { - debug_assert_eq!(self.tag(), TAG_BLOB); + debug_assert!(self.is_blob()); self.check_forwarding_pointer(); self.forward().get_ptr() as *mut Blob } @@ -431,35 +441,96 @@ pub type Tag = usize; // Odd tag numbers are historically expected by the mark-compact GC (for pointer threading). pub const TAG_OBJECT: Tag = 1; pub const TAG_OBJ_IND: Tag = 3; -pub const TAG_ARRAY: Tag = 5; -pub const TAG_BITS64: Tag = 7; -pub const TAG_MUTBOX: Tag = 9; -pub const TAG_CLOSURE: Tag = 11; -pub const TAG_SOME: Tag = 13; -pub const TAG_VARIANT: Tag = 15; -pub const TAG_BLOB: Tag = 17; -pub const TAG_FWD_PTR: Tag = 19; // Used by graph copy stabilization and the copying GC - not to be confused with the incremental GC's forwarding pointer. -pub const TAG_BIGINT: Tag = 21; -pub const TAG_CONCAT: Tag = 23; -pub const TAG_REGION: Tag = 25; -pub const TAG_ONE_WORD_FILLER: Tag = 27; -pub const TAG_FREE_SPACE: Tag = 29; +pub const TAG_ARRAY_I: Tag = 5; // Immutable Array ([T]) +pub const TAG_ARRAY_M: Tag = 7; // Mutable Array ([var T]) +pub const TAG_ARRAY_T: Tag = 9; // Non-nullary Tuple ((T,+)) +pub const TAG_ARRAY_S: Tag = 11; // Shared function pairing TAG_BLOB_A with TAG_BLOB_T (shared ... -> ...) +pub const TAG_BITS64_U: Tag = 13; // Unsigned (Nat64) +pub const TAG_BITS64_S: Tag = 15; // Signed (Int64) +pub const TAG_BITS64_F: Tag = 17; // Float +pub const TAG_MUTBOX: Tag = 19; +pub const TAG_CLOSURE: Tag = 21; +pub const TAG_SOME: Tag = 23; +pub const TAG_VARIANT: Tag = 25; +pub const TAG_BLOB_B: Tag = 27; // Blob of Bytes (Blob) +pub const TAG_BLOB_T: Tag = 29; // Blob of Utf8 (Text) +pub const TAG_BLOB_P: Tag = 31; // Principal (Principal) +pub const TAG_BLOB_A: Tag = 33; // Actor (actor {}) +pub const TAG_FWD_PTR: Tag = 35; // Used by graph copy stabilization and the copying GC - not to be confused with the incremental GC's forwarding pointer. +pub const TAG_BIGINT: Tag = 37; +pub const TAG_CONCAT: Tag = 39; +pub const TAG_REGION: Tag = 41; +pub const TAG_ONE_WORD_FILLER: Tag = 43; +pub const TAG_FREE_SPACE: Tag = 45; // Special value to visit only a range of array fields. // This and all values above it are reserved and mean -// a slice of an array object (i.e. start index) for +// a slice of an array object (i.e. compressed array tag + start index) for // purposes of `visit_pointer_fields`. +// The top two bits encode the original array tag, the remaining bits are the start index of the slice. // Invariant: the value of this (pseudo-)tag must be // higher than all other tags defined above. // Note: The minimum value can be even, as it only denotes // a lower boundary to distinguish slice information from // the actual tag values. -pub const TAG_ARRAY_SLICE_MIN: Tag = 30; +pub const TAG_ARRAY_SLICE_MIN: Tag = 46; +pub const TAG_SPACING: Tag = 2; pub fn is_object_tag(tag: Tag) -> bool { tag >= TAG_OBJECT && tag <= TAG_REGION } +pub fn is_blob_tag(tag: Tag) -> bool { + tag == TAG_BLOB_B || tag == TAG_BLOB_T || tag == TAG_BLOB_P || tag == TAG_BLOB_A +} + +pub fn is_base_array_tag(tag: Tag) -> bool { + tag == TAG_ARRAY_I || tag == TAG_ARRAY_M || tag == TAG_ARRAY_T || tag == TAG_ARRAY_S +} + +pub fn is_array_or_slice_tag(tag: Tag) -> bool { + is_base_array_tag(tag) || tag >= TAG_ARRAY_SLICE_MIN +} + +#[inline] +pub fn start_of_slice(tag: Tag) -> usize { + tag << 2 >> 2 +} + +#[inline] +pub fn tag_of_slice(tag: Tag) -> Tag { + TAG_ARRAY_I + (tag >> (usize::BITS - 2)) * TAG_SPACING +} + +pub fn slice_tag(array_tag: Tag, slice_start: usize) -> Tag { + debug_assert!(is_base_array_tag(array_tag)); + debug_assert!( + slice_start >= TAG_ARRAY_SLICE_MIN && slice_start <= MAX_ARRAY_LENGTH_FOR_ITERATOR + ); + debug_assert!((array_tag - TAG_ARRAY_I) % TAG_SPACING == 0); + (((array_tag - TAG_ARRAY_I) / TAG_SPACING) << (usize::BITS - 2)) | slice_start +} + +pub fn slice_start(tag: Tag) -> (Tag, usize) { + debug_assert!(is_array_or_slice_tag(tag)); + if tag >= TAG_ARRAY_SLICE_MIN { + (tag_of_slice(tag), start_of_slice(tag)) + } else { + (tag, 0) + } +} + +pub fn base_array_tag(tag: Tag) -> Tag { + debug_assert!(is_array_or_slice_tag(tag)); + let base = if tag >= TAG_ARRAY_SLICE_MIN { + tag_of_slice(tag) + } else { + tag + }; + debug_assert!(is_base_array_tag(base)); + base +} + // Common parts of any object. Other object pointers can be coerced into a pointer to this. #[repr(C)] // See the note at the beginning of this module pub struct Obj { @@ -487,7 +558,7 @@ impl Obj { } pub unsafe fn as_blob(self: *mut Self) -> *mut Blob { - debug_assert_eq!(self.tag(), TAG_BLOB); + debug_assert!(is_blob_tag(self.tag())); self as *mut Blob } @@ -558,6 +629,24 @@ impl Array { pub unsafe fn len(self: *const Self) -> usize { (*self).len } + + pub unsafe fn base_tag(self: *const Self) -> Tag { + base_array_tag((*self).header.tag) + } + + pub unsafe fn get_slice_start(self: *const Self) -> (Tag, usize) { + slice_start((*self).header.tag) + } + + pub unsafe fn set_slice_start(self: *mut Self, array_tag: Tag, start: usize) { + debug_assert!(is_base_array_tag(array_tag)); + (*self).header.tag = slice_tag(array_tag, start) + } + + pub unsafe fn restore_tag(self: *mut Self, array_tag: Tag) { + debug_assert!(is_base_array_tag(array_tag)); + (*self).header.tag = array_tag; + } } #[rustfmt::skip] @@ -839,13 +928,13 @@ pub(crate) unsafe fn block_size(address: usize) -> Words { // `block_size` is not used during the incremental mark phase and // therefore, does not support array slicing. - TAG_ARRAY => { + TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S => { let array = address as *mut Array; let size = array.len(); size_of::() + Words(size) } - TAG_BITS64 => size_of::(), + TAG_BITS64_U | TAG_BITS64_S | TAG_BITS64_F => size_of::(), TAG_MUTBOX => size_of::(), @@ -859,7 +948,7 @@ pub(crate) unsafe fn block_size(address: usize) -> Words { TAG_VARIANT => size_of::(), - TAG_BLOB => { + TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P | TAG_BLOB_A => { let blob = address as *mut Blob; size_of::() + blob.len().to_words() } diff --git a/rts/motoko-rts/src/visitor.rs b/rts/motoko-rts/src/visitor.rs index 7d6deb49b3c..14f7cc0abaa 100644 --- a/rts/motoko-rts/src/visitor.rs +++ b/rts/motoko-rts/src/visitor.rs @@ -43,9 +43,10 @@ pub unsafe fn visit_pointer_fields( } } - TAG_ARRAY | TAG_ARRAY_SLICE_MIN.. => { - let slice_start = if tag >= TAG_ARRAY_SLICE_MIN { tag } else { 0 }; + TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_S | TAG_ARRAY_SLICE_MIN.. => { + let (_, slice_start) = slice_start(tag); let array = obj as *mut Array; + debug_assert!(slice_start <= array.len()); let array_payload = array.payload_addr(); let stop = visit_field_range(ctx, slice_start, array); debug_assert!(stop <= array.len()); @@ -120,7 +121,8 @@ pub unsafe fn visit_pointer_fields( } } - TAG_BITS64 | TAG_BLOB | TAG_BIGINT => { + TAG_BITS64_U | TAG_BITS64_S | TAG_BITS64_F | TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P + | TAG_BLOB_A | TAG_BIGINT => { // These don't have pointers, skip } diff --git a/src/codegen/compile.ml b/src/codegen/compile.ml index a1b24ba4571..3e9ca106993 100644 --- a/src/codegen/compile.ml +++ b/src/codegen/compile.ml @@ -240,6 +240,7 @@ module Const = struct | Bool of bool | Word64 of Type.prim * int64 | Float64 of Numerics.Float.t + | Text of string | Blob of string | Null @@ -249,6 +250,7 @@ module Const = struct | Word64 (tyi, i), Word64 (tyj, j) -> tyi = tyj && i = j | Float64 i, Float64 j -> i = j | Bool i, Bool j -> i = j + | Text s, Text t | Blob s, Blob t -> s = t | Null, Null -> true | _ -> false @@ -288,7 +290,8 @@ module Const = struct | Message of int32 (* anonymous message, only temporary *) | Obj of (string * v) list | Unit - | Array of v list (* also tuples, but not nullary *) + | Array of v list (* immutable arrays *) + | Tuple of v list (* non-nullary tuples *) | Tag of (string * v) | Opt of v | Lit of lit @@ -302,12 +305,15 @@ module Const = struct | Unit, Unit -> true | Array elements1, Array elements2 -> List.for_all2 eq elements1 elements2 + | Tuple elements1, Tuple elements2 -> + List.for_all2 eq elements1 elements2 | Tag (name1, tag_value1), Tag (name2, tag_value2) -> (name1 = name2) && (eq tag_value1 tag_value2) | Opt opt_value1, Opt opt_value2 -> eq opt_value1 opt_value2 | Lit l1, Lit l2 -> lit_eq l1 l2 | Fun _, _ | Message _, _ | Obj _, _ | Unit, _ - | Array _, _ | Tag _, _ | Opt _, _ | Lit _, _ -> false + | Array _, _ | Tuple _, _ | Tag _, _ | Opt _, _ + | Lit _, _ -> false end (* Const *) @@ -1255,8 +1261,8 @@ module RTS = struct E.add_func_import env "rts" "alloc_words" [I64Type] [I64Type]; E.add_func_import env "rts" "get_total_allocations" [] [I64Type]; E.add_func_import env "rts" "get_heap_size" [] [I64Type]; - E.add_func_import env "rts" "alloc_blob" [I64Type] [I64Type]; - E.add_func_import env "rts" "alloc_array" [I64Type] [I64Type]; + E.add_func_import env "rts" "alloc_blob" [I64Type; I64Type] [I64Type]; + E.add_func_import env "rts" "alloc_array" [I64Type; I64Type] [I64Type]; E.add_func_import env "rts" "read_persistence_version" [] [I64Type]; E.add_func_import env "rts" "stop_gc_before_stabilization" [] []; E.add_func_import env "rts" "start_gc_after_destabilization" [] []; @@ -1881,16 +1887,31 @@ module Tagged = struct so update all! *) + type bits_sort = + | U (* signed *) + | S (* unsigned *) + | F (* float *) + type array_sort = + | I (* [ T ] *) + | M (* [var T ] *) + | T (* (T,+) *) + | S (* shared ... -> ... *) + type blob_sort = + | B (* Blob *) + | T (* Text *) + | P (* Principal *) + | A (* actor { ... } *) + type [@warning "-37"] tag = | Object | ObjInd (* The indirection used for object fields *) - | Array (* Also a tuple *) - | Bits64 (* Contains a 64 bit number *) + | Array of array_sort (* Also a tuple *) + | Bits64 of bits_sort (* Contains a 64 bit number *) | MutBox (* used for mutable heap-allocated variables *) | Closure | Some (* For opt *) | Variant - | Blob + | Blob of blob_sort | Indirection (* Only used by the GC *) | BigInt | Concat (* String concatenation, used by rts/text.c *) @@ -1909,20 +1930,28 @@ module Tagged = struct let int_of_tag = function | Object -> 1L | ObjInd -> 3L - | Array -> 5L - | Bits64 -> 7L - | MutBox -> 9L - | Closure -> 11L - | Some -> 13L - | Variant -> 15L - | Blob -> 17L - | Indirection -> 19L - | BigInt -> 21L - | Concat -> 23L - | Region -> 25L - | OneWordFiller -> 27L - | FreeSpace -> 29L - | ArraySliceMinimum -> 30L + | Array I -> 5L + | Array M -> 7L + | Array T -> 9L + | Array S -> 11L + | Bits64 U -> 13L + | Bits64 S -> 15L + | Bits64 F -> 17L + | MutBox -> 19L + | Closure -> 21L + | Some -> 23L + | Variant -> 25L + | Blob B -> 27L + | Blob T -> 29L + | Blob P -> 31L + | Blob A -> 33L + | Indirection -> 35L + | BigInt -> 37L + | Concat -> 39L + | Region -> 41L + | OneWordFiller -> 43L + | FreeSpace -> 45L + | ArraySliceMinimum -> 46L (* Next two tags won't be seen by the GC, so no need to set the lowest bit for `CoercionFailure` and `StableSeen` *) | CoercionFailure -> 0xffff_ffff_ffff_fffeL @@ -1981,6 +2010,23 @@ module Tagged = struct load_forwarding_pointer env ^^ Heap.load_field tag_field + let sanity_check_tag line env tag = + let tag = int_of_tag tag in + let name = "sanity_check_tag_" ^ Int64.to_string tag ^ + (if TaggingScheme.debug then Int.to_string line else "") + in + if TaggingScheme.debug || !Flags.sanity then + Func.share_code1 Func.Always env name ("obj", I64Type) [I64Type] + (fun env get_obj -> + get_obj ^^ + load_tag env ^^ + compile_unboxed_const tag ^^ + compile_comparison I64Op.Eq ^^ + E.else_trap_with env name ^^ + get_obj) + else + G.nop + let check_forwarding env unskewed = let name = "check_forwarding_" ^ if unskewed then "unskewed" else "skewed" in Func.share_code1 Func.Always env name ("object", I64Type) [I64Type] (fun env get_object -> @@ -2038,62 +2084,6 @@ module Tagged = struct set_tag ^^ go cases - (* like branch_default but also pushes the scrutinee on the stack for the - * branch's consumption *) - let _branch_default_with env retty def cases = - let (set_o, get_o) = new_local env "o" in - let prep (t, code) = (t, get_o ^^ code) - in set_o ^^ get_o ^^ branch_default env retty def (List.map prep cases) - - (* like branch_default_with but the tag is known statically *) - let branch_with env retty = function - | [] -> G.i Unreachable - | [_, code] -> code - | (_, code) :: cases -> - let (set_o, get_o) = new_local env "o" in - let prep (t, code) = (t, get_o ^^ code) - in set_o ^^ get_o ^^ branch_default env retty (get_o ^^ code) (List.map prep cases) - - (* Can a value of this type be represented by a heap object with this tag? *) - (* Needs to be conservative, i.e. return `true` if unsure *) - (* This function can also be used as assertions in a lint mode, e.g. in compile_exp *) - let can_have_tag ty tag = - let open Mo_types.Type in - match (tag : tag) with - | Region -> - begin match normalize ty with - | (Con _ | Any) -> true - | (Prim Region) -> true - | (Prim _ | Obj _ | Array _ | Tup _ | Opt _ | Variant _ | Func _ | Non) -> false - | (Pre | Async _ | Mut _ | Var _ | Typ _) -> assert false - end - | Array -> - begin match normalize ty with - | (Con _ | Any) -> true - | (Array _ | Tup _) -> true - | (Prim _ | Obj _ | Opt _ | Variant _ | Func _ | Non) -> false - | (Pre | Async _ | Mut _ | Var _ | Typ _) -> assert false - end - | Blob -> - begin match normalize ty with - | (Con _ | Any) -> true - | (Prim (Text|Blob|Principal)) -> true - | (Prim _ | Obj _ | Array _ | Tup _ | Opt _ | Variant _ | Func _ | Non) -> false - | (Pre | Async _ | Mut _ | Var _ | Typ _) -> assert false - end - | Object -> - begin match normalize ty with - | (Con _ | Any) -> true - | (Obj _) -> true - | (Prim _ | Array _ | Tup _ | Opt _ | Variant _ | Func _ | Non) -> false - | (Pre | Async _ | Mut _ | Var _ | Typ _) -> assert false - end - | _ -> true - - (* like branch_with but with type information to statically skip some branches *) - let _branch_typed_with env ty retty branches = - branch_with env retty (List.filter (fun (tag,c) -> can_have_tag ty tag) branches) - let allocation_barrier env = E.call_import env "rts" "allocation_barrier" @@ -2372,10 +2362,14 @@ module BoxedWord64 = struct The object header includes the object tag (Bits64) and the forwarding pointer. *) - let heap_tag env pty = Tagged.Bits64 - let payload_field = Tagged.header_size + let heap_tag env pty = + match pty with + | Type.Nat64 -> Tagged.(Bits64 U) + | Type.Int64 -> Tagged.(Bits64 S) + | _ -> assert false + let compile_box env pty compile_elem : G.t = let (set_i, get_i) = new_local env "boxed_i64" in let size = 4L in @@ -2406,7 +2400,10 @@ module BoxedWord64 = struct get_n ^^ BitTagged.if_tagged_scalar env [I64Type] (get_n ^^ BitTagged.untag __LINE__ env pty) - (get_n ^^ Tagged.load_forwarding_pointer env ^^ Tagged.load_field env payload_field) + (get_n ^^ + Tagged.load_forwarding_pointer env ^^ + Tagged.(sanity_check_tag __LINE__ env (heap_tag env pty)) ^^ + Tagged.load_field env payload_field) ) end (* BoxedWord64 *) @@ -2736,14 +2733,17 @@ module Float = struct let box env = Func.share_code1 Func.Never env "box_f64" ("f", F64Type) [I64Type] (fun env get_f -> let (set_i, get_i) = new_local env "boxed_f64" in let size = Int64.add Tagged.header_size 2L in - Tagged.alloc env size Tagged.Bits64 ^^ + Tagged.alloc env size Tagged.(Bits64 F) ^^ set_i ^^ get_i ^^ get_f ^^ Tagged.store_field_float64 env payload_field ^^ get_i ^^ Tagged.allocation_barrier env ) - let unbox env = Tagged.load_forwarding_pointer env ^^ Tagged.load_field_float64 env payload_field + let unbox env = + Tagged.load_forwarding_pointer env ^^ + Tagged.(sanity_check_tag __LINE__ env (Bits64 F)) ^^ + Tagged.load_field_float64 env payload_field let constant env f = Tagged.shared_object env (fun env -> compile_unboxed_const f ^^ @@ -3688,7 +3688,9 @@ module Blob = struct BigNum.from_word64 env ) - let alloc env = + let alloc env sort len = + compile_unboxed_const Tagged.(int_of_tag (Blob sort)) ^^ + len ^^ E.call_import env "rts" "alloc_blob" ^^ (* uninitialized blob payload is allowed by the barrier *) Tagged.allocation_barrier env @@ -3699,25 +3701,24 @@ module Blob = struct Tagged.load_forwarding_pointer env ^^ compile_add_const (unskewed_payload_offset env) - let load_data_segment env segment_index data_length = + let load_data_segment env sort segment_index data_length = let (set_blob, get_blob) = new_local env "data_segment_blob" in - data_length ^^ - alloc env ^^ set_blob ^^ + alloc env sort data_length ^^ set_blob ^^ get_blob ^^ payload_ptr_unskewed env ^^ (* target address *) compile_const_32 0l ^^ (* data offset *) data_length ^^ G.i (Convert (Wasm_exts.Values.I32 I32Op.WrapI64)) ^^ G.i (MemoryInit (nr segment_index)) ^^ get_blob - let constant env payload = + let constant env sort payload = Tagged.shared_object env (fun env -> let blob_length = Int64.of_int (String.length payload) in let segment_index = E.add_static env StaticBytes.[Bytes payload] in - load_data_segment env segment_index (compile_unboxed_const blob_length) + load_data_segment env sort segment_index (compile_unboxed_const blob_length) ) - let lit env payload = - Tagged.materialize_shared_value env (constant env payload) + let lit env sort payload = + Tagged.materialize_shared_value env (constant env sort payload) let as_ptr_len env = Func.share_code1 Func.Never env "as_ptr_size" ("x", I64Type) [I64Type; I64Type] ( fun env get_x -> @@ -3725,14 +3726,13 @@ module Blob = struct get_x ^^ len env ) - let lit_ptr_len env s = - lit env s ^^ + let lit_ptr_len env sort s = + lit env sort s ^^ as_ptr_len env - let load_data_segment env segment_index data_length = + let load_data_segment env sort segment_index data_length = let (set_blob, get_blob) = new_local env "data_segment_blob" in - data_length ^^ - alloc env ^^ set_blob ^^ + alloc env sort data_length ^^ set_blob ^^ get_blob ^^ payload_ptr_unskewed env ^^ (* target address *) compile_const_32 0l ^^ (* data offset *) data_length ^^ G.i (Convert (Wasm_exts.Values.I32 I32Op.WrapI64)) ^^ @@ -3742,7 +3742,7 @@ module Blob = struct let of_ptr_size env = Func.share_code2 Func.Always env "blob_of_ptr_size" (("ptr", I64Type), ("size" , I64Type)) [I64Type] ( fun env get_ptr get_size -> let (set_x, get_x) = new_local env "x" in - get_size ^^ alloc env ^^ set_x ^^ + alloc env Tagged.B get_size ^^ set_x ^^ get_x ^^ payload_ptr_unskewed env ^^ get_ptr ^^ get_size ^^ @@ -3750,12 +3750,28 @@ module Blob = struct get_x ) - let of_size_copy env get_size_fun copy_fun offset_fun = + let copy env src_sort dst_sort = + let name = Printf.sprintf "blob_copy_%s_%s" + (Int64.to_string (Tagged.int_of_tag (Tagged.Blob src_sort))) + (Int64.to_string (Tagged.int_of_tag (Tagged.Blob dst_sort))) + in + Func.share_code1 Func.Never env name ("src", I64Type) [I64Type] ( + fun env get_src -> + let (set_dst, get_dst) = new_local env "dst" in + alloc env dst_sort (get_src ^^ len env) ^^ set_dst ^^ + get_dst ^^ payload_ptr_unskewed env ^^ + get_src ^^ Tagged.sanity_check_tag __LINE__ env (Tagged.Blob src_sort) ^^ + as_ptr_len env ^^ + Heap.memcpy env ^^ + get_dst + ) + + let of_size_copy env sort get_size_fun copy_fun offset_fun = let (set_len, get_len) = new_local env "len" in let (set_blob, get_blob) = new_local env "blob" in get_size_fun env ^^ set_len ^^ - get_len ^^ alloc env ^^ set_blob ^^ + alloc env sort get_len ^^ set_blob ^^ get_blob ^^ payload_ptr_unskewed env ^^ offset_fun env ^^ get_len ^^ @@ -3872,7 +3888,10 @@ module Blob = struct E.call_import env "rts" "blob_iter_next" ^^ TaggedSmallWord.msb_adjust Type.Nat8 - let dyn_alloc_scratch env = alloc env ^^ payload_ptr_unskewed env + let dyn_alloc_scratch env = + let (set_len, get_len) = new_local env "len" in + set_len ^^ + alloc env Tagged.B get_len ^^ payload_ptr_unskewed env end (* Blob *) @@ -3942,7 +3961,7 @@ module Object = struct List.sort compare in let hash_blob = let hash_payload = StaticBytes.[ i64s hashes ] in - Blob.constant env (StaticBytes.as_bytes hash_payload) + Blob.constant env Tagged.B (StaticBytes.as_bytes hash_payload) in (fun env -> @@ -4212,7 +4231,11 @@ module Text = struct get_blob ^^ Blob.as_ptr_len env ^^ E.call_import env "rts" "utf8_valid" ^^ Bool.from_rts_int32 ^^ - E.if1 I64Type (Opt.inject_simple env get_blob) (Opt.null_lit env) + E.if1 I64Type + (get_blob ^^ Blob.copy env Tagged.B Tagged.T ^^ + set_blob ^^ + Opt.inject_simple env get_blob) + (Opt.null_lit env) let iter env = E.call_import env "rts" "text_iter" @@ -4312,7 +4335,7 @@ module Arr = struct Func.share_code2 Func.Never env "Array.idx_bigint" (("array", I64Type), ("idx", I64Type)) [I64Type] (fun env get_array get_idx -> get_array ^^ get_idx ^^ - BigNum.to_word64_with env (Blob.lit env "Array index out of bounds") ^^ + BigNum.to_word64_with env (Blob.lit env Tagged.T "Array index out of bounds") ^^ idx env ) @@ -4321,20 +4344,22 @@ module Arr = struct | _ -> assert false (* Compile an array literal. *) - let lit env element_instructions = - Tagged.obj env Tagged.Array + let lit env sort element_instructions = + Tagged.obj env Tagged.(Array sort) ([ compile_unboxed_const (Wasm.I64.of_int_u (List.length element_instructions)) ] @ element_instructions) - let constant env elements = + let constant env sort elements = Tagged.shared_object env (fun env -> let materialized_elements = List.map (fun element -> Tagged.materialize_shared_value env element) elements in - lit env materialized_elements + lit env sort materialized_elements ) (* Does not initialize the fields! *) (* Note: Post allocation barrier must be applied after initialization *) - let alloc env = + let alloc env array_sort len = + compile_unboxed_const Tagged.(int_of_tag (Array array_sort)) ^^ + len ^^ E.call_import env "rts" "alloc_array" let iterate env get_array body = @@ -4382,7 +4407,8 @@ module Arr = struct (* Allocate *) BigNum.to_word64 env ^^ - alloc env ^^ + set_r ^^ + alloc env Tagged.M get_r ^^ set_r ^^ (* Write elements *) @@ -4404,7 +4430,8 @@ module Arr = struct (* Allocate *) BigNum.to_word64 env ^^ - alloc env ^^ + set_r ^^ + alloc env Tagged.I get_r ^^ set_r ^^ (* Initial index *) @@ -4434,14 +4461,15 @@ module Arr = struct get_r ^^ Tagged.allocation_barrier env - let ofBlob env = - Func.share_code1 Func.Always env "Arr.ofBlob" ("blob", I64Type) [I64Type] (fun env get_blob -> + let ofBlob env sort = + let name = Tagged.(match sort with I -> "Arr.ofBlob" | M -> "Arr.ofBlobMut" | _ -> assert false) in + Func.share_code1 Func.Always env name ("blob", I64Type) [I64Type] (fun env get_blob -> let (set_len, get_len) = new_local env "len" in let (set_r, get_r) = new_local env "r" in get_blob ^^ Blob.len env ^^ set_len ^^ - get_len ^^ alloc env ^^ set_r ^^ + alloc env sort get_len ^^ set_r ^^ get_len ^^ from_0_to_n env (fun get_i -> get_r ^^ get_i ^^ unsafe_idx env ^^ @@ -4463,7 +4491,7 @@ module Arr = struct get_a ^^ len env ^^ set_len ^^ - get_len ^^ Blob.alloc env ^^ set_r ^^ + Blob.alloc env Tagged.B get_len ^^ set_r ^^ get_len ^^ from_0_to_n env (fun get_i -> get_r ^^ Blob.payload_ptr_unskewed env ^^ @@ -4495,6 +4523,7 @@ module Tuple = struct (* Expects on the stack the pointer to the array. *) let load_n env n = Tagged.load_forwarding_pointer env ^^ + Tagged.(sanity_check_tag __LINE__ env (Array T)) ^^ Tagged.load_field env (Int64.add Arr.header_size n) (* Takes n elements of the stack and produces an argument tuple *) @@ -4504,7 +4533,7 @@ module Tuple = struct let name = Printf.sprintf "to_%i_tuple" n in let args = Lib.List.table n (fun i -> Printf.sprintf "arg%i" i, I64Type) in Func.share_code Func.Never env name args [I64Type] (fun env getters -> - Arr.lit env (Lib.List.table n (fun i -> List.nth getters i)) + Arr.lit env Tagged.T (Lib.List.table n (fun i -> List.nth getters i)) ) (* Takes an argument tuple and puts the elements on the stack: *) @@ -4665,7 +4694,7 @@ module IC = struct G.i (GlobalSet (nr (E.get_global env "__run_post_upgrade"))) let init_globals env = - Blob.lit env "" ^^ + Blob.lit env Tagged.T "" ^^ set_call_perform_message env let i32s n = Lib.List.make n I32Type @@ -4843,7 +4872,7 @@ module IC = struct (* For debugging *) let _compile_static_print env s = - Blob.lit_ptr_len env s ^^ print_ptr_len env + Blob.lit_ptr_len env Tagged.T s ^^ print_ptr_len env let ic_trap env = Func.share_code2 Func.Always env "ic_trap" (("ptr", I64Type), ("len", I64Type)) [] (fun env get_ptr get_length -> @@ -4859,7 +4888,7 @@ module IC = struct | Flags.ICMode | Flags.RefMode -> ic_trap env ^^ G.i Unreachable let trap_with env s = - Blob.lit_ptr_len env s ^^ trap_ptr_len env + Blob.lit_ptr_len env Tagged.T s ^^ trap_ptr_len env let trap_text env = Text.to_blob env ^^ Blob.as_ptr_len env ^^ trap_ptr_len env @@ -5005,7 +5034,7 @@ module IC = struct match E.mode env with | Flags.ICMode | Flags.RefMode -> Func.share_code0 Func.Never env "canister_self" [I64Type] (fun env -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.A (fun env -> system_call env "canister_self_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -5026,7 +5055,7 @@ module IC = struct let caller env = match E.mode env with | Flags.ICMode | Flags.RefMode -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.P (fun env -> system_call env "msg_caller_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -5039,7 +5068,7 @@ module IC = struct let method_name env = match E.mode env with | Flags.ICMode | Flags.RefMode -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.T (fun env -> system_call env "msg_method_name_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -5052,7 +5081,7 @@ module IC = struct let arg_data env = match E.mode env with | Flags.ICMode | Flags.RefMode -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.B (fun env -> system_call env "msg_arg_data_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -5093,7 +5122,7 @@ module IC = struct let error_message env = Func.share_code0 Func.Never env "error_message" [I64Type] (fun env -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.T (fun env -> system_call env "msg_reject_msg_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) (fun env -> @@ -5118,14 +5147,19 @@ module IC = struct ) let static_nullary_reply env = - Blob.lit_ptr_len env "DIDL\x00\x00" ^^ + Blob.lit_ptr_len env Tagged.B "DIDL\x00\x00" ^^ reply_with_data env (* Actor reference on the stack *) let actor_public_field env name = (* simply tuple canister name and function name *) - Blob.lit env name ^^ - Tuple.from_stack env 2 + Tagged.(sanity_check_tag __LINE__ env (Blob A)) ^^ + Blob.lit env Tagged.T name ^^ + Func.share_code2 Func.Never env "actor_public_field" (("actor", I64Type), ("func", I64Type)) [] ( + fun env get_actor get_func -> + Arr.lit env Tagged.S [get_actor; get_func] + ) + let fail_assert env at = let open Source in @@ -5238,7 +5272,7 @@ module IC = struct E.if1 I64Type begin Opt.inject_simple env ( - Blob.of_size_copy env + Blob.of_size_copy env Tagged.B (fun env -> system_call env "data_certificate_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -5686,7 +5720,7 @@ module StableMem = struct get_offset ^^ get_len ^^ guard_range env ^^ - get_len ^^ Blob.alloc env ^^ set_blob ^^ + Blob.alloc env Tagged.B get_len ^^ set_blob ^^ get_blob ^^ Blob.payload_ptr_unskewed env ^^ get_offset ^^ get_len ^^ @@ -6153,7 +6187,7 @@ module Serialization = struct get_data_size ^^ compile_add_const header_size ^^ Blob.dyn_alloc_scratch env ^^ set_data_buf ^^ get_data_buf ^^ - Blob.lit env header ^^ Blob.payload_ptr_unskewed env ^^ + Blob.lit env Tagged.B header ^^ Blob.payload_ptr_unskewed env ^^ compile_unboxed_const header_size ^^ Heap.memcpy env ^^ get_data_buf ^^ compile_add_const header_size ^^ set_data_buf @@ -6263,19 +6297,19 @@ module Serialization = struct let get_global_candid_data env = Tagged.share env (fun env -> let descriptor = get_global_type_descriptor env in - Blob.load_data_segment env E.(descriptor.candid_data_segment) (get_candid_data_length env) + Blob.load_data_segment env Tagged.B E.(descriptor.candid_data_segment) (get_candid_data_length env) ) let get_global_type_offsets env = Tagged.share env (fun env -> let descriptor = get_global_type_descriptor env in - Blob.load_data_segment env E.(descriptor.type_offsets_segment) (get_type_offsets_length env) + Blob.load_data_segment env Tagged.B E.(descriptor.type_offsets_segment) (get_type_offsets_length env) ) let get_global_idl_types env = Tagged.share env (fun env -> let descriptor = get_global_type_descriptor env in - Blob.load_data_segment env E.(descriptor.idl_types_segment) (get_idl_types_length env) + Blob.load_data_segment env Tagged.B E.(descriptor.idl_types_segment) (get_idl_types_length env) ) module Registers = struct @@ -6647,7 +6681,7 @@ module Serialization = struct compile_comparison I64Op.GeU ^^ G.i (Binary (Wasm_exts.Values.I64 I64Op.And)) ^^ E.if1 I64Type begin - (compile_unboxed_const Tagged.(int_of_tag Array)) + (compile_unboxed_const Tagged.(int_of_tag (Array M))) end begin get_temp end @@ -6663,7 +6697,7 @@ module Serialization = struct G.i (Binary (Wasm_exts.Values.I64 I64Op.Or)) ^^ get_tag ^^ compile_eq_const Tagged.(int_of_tag ObjInd) ^^ G.i (Binary (Wasm_exts.Values.I64 I64Op.Or)) ^^ - get_tag ^^ compile_eq_const Tagged.(int_of_tag Array) ^^ + get_tag ^^ compile_eq_const Tagged.(int_of_tag (Array M)) ^^ G.i (Binary (Wasm_exts.Values.I64 I64Op.Or)) ^^ get_tag ^^ compile_eq_const Tagged.(int_of_tag Region) ^^ G.i (Binary (Wasm_exts.Values.I64 I64Op.Or)) ^^ @@ -6818,7 +6852,7 @@ module Serialization = struct E.then_trap_with env "unvisited mutable data in serialize_go (MutBox)" ^^ get_tag ^^ compile_eq_const Tagged.(int_of_tag ObjInd) ^^ E.then_trap_with env "unvisited mutable data in serialize_go (ObjInd)" ^^ - get_tag ^^ compile_eq_const Tagged.(int_of_tag Array) ^^ + get_tag ^^ compile_eq_const Tagged.(int_of_tag (Array M)) ^^ E.then_trap_with env "unvisited mutable data in serialize_go (Array)" ^^ get_tag ^^ compile_eq_const Tagged.(int_of_tag Region) ^^ E.then_trap_with env "unvisited mutable data in serialize_go (Region)" ^^ @@ -7137,13 +7171,13 @@ module Serialization = struct let (set_x, get_x) = new_local env "x" in ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^ - get_len ^^ Blob.alloc env ^^ set_x ^^ + Blob.alloc env Tagged.B get_len ^^ set_x ^^ get_x ^^ Blob.payload_ptr_unskewed env ^^ ReadBuf.read_blob env get_data_buf get_len ^^ get_x in - let read_principal () = + let read_principal sort () = let (set_len, get_len) = new_local env "len" in let (set_x, get_x) = new_local env "x" in ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^ @@ -7154,7 +7188,7 @@ module Serialization = struct get_len ^^ compile_unboxed_const 29L ^^ compile_comparison I64Op.LeU ^^ E.else_trap_with env "IDL error: principal too long" ^^ - get_len ^^ Blob.alloc env ^^ set_x ^^ + Blob.alloc env sort get_len ^^ set_x ^^ get_x ^^ Blob.payload_ptr_unskewed env ^^ ReadBuf.read_blob env get_data_buf get_len ^^ get_x @@ -7175,7 +7209,7 @@ module Serialization = struct let read_actor_data () = read_byte_tagged [ E.trap_with env "IDL error: unexpected actor reference" - ; read_principal () + ; read_principal Tagged.A () ] in @@ -7370,11 +7404,11 @@ module Serialization = struct *) get_thing ^^ set_result ^^ get_memo ^^ get_result ^^ write_compressed_pointer env ^^ - get_memo ^^ compile_add_const 4L ^^ Blob.lit env (typ_hash t) ^^ write_compressed_pointer env + get_memo ^^ compile_add_const 4L ^^ Blob.lit env Tagged.B (typ_hash t) ^^ write_compressed_pointer env ) end begin (* Decoded before. Check type hash *) - read_compressed_pointer env get_data_buf ^^ Blob.lit env (typ_hash t) ^^ + read_compressed_pointer env get_data_buf ^^ Blob.lit env Tagged.B (typ_hash t) ^^ Blob.compare env (Some Operator.EqOp) ^^ E.else_trap_with env ("Stable memory error: Aliased at wrong type, expected: " ^ typ_hash t) end ^^ @@ -7469,7 +7503,7 @@ module Serialization = struct begin read_byte_tagged [ E.trap_with env "IDL error: unexpected principal reference" - ; read_principal () + ; read_principal Tagged.P () ] end | Prim Text -> @@ -7545,7 +7579,7 @@ module Serialization = struct *) with_composite_arg_typ get_array_typ idl_vec (ReadBuf.read_sleb128 env) ^^ set_arg_typ ^^ ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^ - get_len ^^ Arr.alloc env ^^ set_x ^^ + Arr.alloc env Tagged.M get_len ^^ set_x ^^ on_alloc get_x ^^ get_len ^^ from_0_to_n env (fun get_i -> get_x ^^ get_i ^^ Arr.unsafe_idx env ^^ @@ -7567,7 +7601,7 @@ module Serialization = struct (* pre-allocate a region object, with dummy fields *) compile_unboxed_const 0L ^^ (* id *) compile_unboxed_const 0L ^^ (* pagecount *) - Blob.lit env "" ^^ (* vec_pages *) + Blob.lit env Tagged.B "" ^^ (* vec_pages *) Region.alloc_region env ^^ set_region ^^ on_alloc get_region ^^ @@ -7587,7 +7621,7 @@ module Serialization = struct ReadBuf.read_sleb128 env get_typ_buf ^^ set_arg_typ ^^ ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^ - get_len ^^ Arr.alloc env ^^ set_x ^^ + Arr.alloc env Tagged.I get_len ^^ set_x ^^ get_len ^^ from_0_to_n env (fun get_i -> get_x ^^ get_i ^^ Arr.unsafe_idx env ^^ get_arg_typ ^^ go env t ^^ set_val ^^ @@ -7693,9 +7727,11 @@ module Serialization = struct (with_composite_typ idl_func (fun _get_typ_buf -> read_byte_tagged [ E.trap_with env "IDL error: unexpected function reference" - ; read_actor_data () ^^ - read_text () ^^ - Tuple.from_stack env 2 + ; let (set_actor, get_actor) = new_local env "actor" in + let (set_func, get_func) = new_local env "func" in + read_actor_data () ^^ set_actor ^^ + read_text () ^^ set_func ^^ + Arr.lit env Tagged.S [get_actor; get_func] ])) (skip get_idltyp ^^ coercion_failed "IDL error: incompatible function type") @@ -8275,7 +8311,7 @@ module OldStabilization = struct let (set_blob, get_blob) = new_local env "blob" in (* read blob from stable memory *) - get_len ^^ Blob.alloc env ^^ set_blob ^^ + Blob.alloc env Tagged.B get_len ^^ set_blob ^^ get_blob ^^ Blob.payload_ptr_unskewed env ^^ get_offset ^^ get_len ^^ @@ -8474,8 +8510,8 @@ module EnhancedOrthogonalPersistence = struct let (candid_type_desc, type_offsets, type_indices) = Serialization.(type_desc env Persistence [actor_type]) in let serialized_offsets = StaticBytes.(as_bytes [i64s (List.map Int64.of_int type_offsets)]) in assert (type_indices = [0l]); - Blob.lit env candid_type_desc ^^ - Blob.lit env serialized_offsets + Blob.lit env Tagged.B candid_type_desc ^^ + Blob.lit env Tagged.B serialized_offsets let register_stable_type env actor_type = create_type_descriptor env actor_type ^^ @@ -8567,7 +8603,7 @@ end module GCRoots = struct let register_static_variables env = E.(env.object_pool.frozen) := true; - Func.share_code0 Func.Always env "initalize_root_array" [] (fun env -> + Func.share_code0 Func.Always env "initialize_root_array" [] (fun env -> let length = Int64.of_int (E.object_pool_size env) in compile_unboxed_const length ^^ E.call_import env "rts" "initialize_static_variables" ^^ @@ -8656,7 +8692,8 @@ module StackRep = struct let rec build_constant env = function | Const.Lit (Const.Vanilla value) -> E.Vanilla value | Const.Lit (Const.Bool number) -> E.Vanilla (Bool.vanilla_lit number) - | Const.Lit (Const.Blob payload) -> Blob.constant env payload + | Const.Lit (Const.Text payload) -> Blob.constant env Tagged.T payload + | Const.Lit (Const.Blob payload) -> Blob.constant env Tagged.B payload | Const.Lit (Const.Null) -> E.Vanilla Opt.null_vanilla_lit | Const.Lit (Const.BigInt number) -> BigNum.constant env number | Const.Lit (Const.Word64 (pty, number)) -> BoxedWord64.constant env pty number @@ -8673,7 +8710,10 @@ module StackRep = struct ) | Const.Array elements -> let constant_elements = List.map (build_constant env) elements in - Arr.constant env constant_elements + Arr.constant env Tagged.I constant_elements + | Const.Tuple elements -> + let constant_elements = List.map (build_constant env) elements in + Arr.constant env Tagged.T constant_elements | Const.Obj fields -> let constant_fields = List.map (fun (name, value) -> (name, build_constant env value)) fields in Object.constant env constant_fields @@ -8715,7 +8755,7 @@ module StackRep = struct compile_unboxed_const n | Const Const.Lit (Const.Float64 f), UnboxedFloat64 -> Float.compile_unboxed_const f | Const c, UnboxedTuple 0 -> G.nop - | Const Const.Array cs, UnboxedTuple n -> + | Const Const.Tuple cs, UnboxedTuple n -> assert (n = List.length cs); G.concat_map (fun c -> materialize_constant env c) cs | _, _ -> @@ -9219,7 +9259,7 @@ module FuncDec = struct (1, "@callback", (fun env -> - Blob.of_size_copy env + Blob.of_size_copy env Tagged.B (fun env -> IC.system_call env "msg_arg_data_size" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32))) @@ -9275,7 +9315,7 @@ module FuncDec = struct the first and second must be the reply and reject continuations. *) fun closure_getters -> let (set_cb_index, get_cb_index) = new_local32 env "cb_index" in - Arr.lit env closure_getters ^^ + Arr.lit env Tagged.T closure_getters ^^ ContinuationTable.remember env ^^ G.i (Convert (Wasm_exts.Values.I32 I32Op.WrapI64)) ^^ set_cb_index ^^ @@ -9335,7 +9375,7 @@ module FuncDec = struct IC.system_call env "call_perform" ^^ G.i (Convert (Wasm_exts.Values.I64 I64Op.ExtendUI32)) ^^ IC.set_call_perform_status env ^^ - Blob.lit env message ^^ + Blob.lit env Tagged.T message ^^ IC.set_call_perform_message env ^^ IC.get_call_perform_status env ^^ compile_unboxed_const 0L ^^ @@ -9413,11 +9453,11 @@ module FuncDec = struct G.i Drop ^^ compile_unboxed_zero ^^ IC.set_call_perform_status env ^^ - Blob.lit env "" ^^ + Blob.lit env Tagged.T "" ^^ IC.set_call_perform_message env else IC.set_call_perform_status env ^^ - Blob.lit env "could not perform oneway" ^^ + Blob.lit env Tagged.T "could not perform oneway" ^^ IC.set_call_perform_message env) | _ -> assert false @@ -9554,7 +9594,7 @@ module IncrementalGraphStabilization = struct let call_async_stabilization env = IC.get_self_reference env ^^ Blob.as_ptr_len env ^^ - Blob.lit_ptr_len env async_stabilization_method_name ^^ + Blob.lit_ptr_len env Tagged.T async_stabilization_method_name ^^ compile_const_32 (async_stabilization_reply_callback env) ^^ compile_const_32 0l ^^ compile_const_32 (async_stabilization_reject_callback env) ^^ compile_const_32 0l ^^ IC.system_call env "call_new_64" ^^ @@ -9662,7 +9702,7 @@ module IncrementalGraphStabilization = struct let call_async_destabilization env = IC.get_self_reference env ^^ Blob.as_ptr_len env ^^ - Blob.lit_ptr_len env async_destabilization_method_name ^^ + Blob.lit_ptr_len env Tagged.T async_destabilization_method_name ^^ compile_const_32 (async_destabilization_reply_callback env) ^^ compile_const_32 0l ^^ compile_const_32 (async_destabilization_reject_callback env) ^^ compile_const_32 0l ^^ IC.system_call env "call_new_64" ^^ @@ -10143,7 +10183,7 @@ let const_lit_of_lit : Ir.lit -> Const.lit = function | Nat64Lit n -> Const.Word64 (Type.Nat64, (Big_int.int64_of_big_int (nat64_to_int64 (Numerics.Nat64.to_big_int n)))) | CharLit c -> Const.Vanilla (TaggedSmallWord.vanilla_lit Type.Char c) | NullLit -> Const.Null - | TextLit t + | TextLit t -> Const.Text t | BlobLit t -> Const.Blob t | FloatLit f -> Const.Float64 f @@ -10901,7 +10941,7 @@ and compile_prim_invocation (env : E.t) ae p es at = | ArrayPrim (m, t), es -> SR.Vanilla, - Arr.lit env (List.map (compile_exp_vanilla env ae) es) + Arr.lit env Tagged.(if m = Ir.Var then M else I) (List.map (compile_exp_vanilla env ae) es) | IdxPrim, [e1; e2] -> SR.Vanilla, compile_array_index env ae e1 e2 ^^ @@ -11184,7 +11224,7 @@ and compile_prim_invocation (env : E.t) ae p es at = (Opt.null_lit env) (go ls') | [] -> - Opt.inject env (Arr.lit env locals) + Opt.inject env (Arr.lit env Tagged.T locals) in go locals) end @@ -11465,7 +11505,7 @@ and compile_prim_invocation (env : E.t) ae p es at = compile_exp_as env ae SR.Vanilla e0 ^^ compile_exp_as env ae (SR.UnboxedWord64 Type.Nat64) e1 ^^ compile_exp_as env ae SR.Vanilla e2 ^^ - BigNum.to_word64_with env (Blob.lit env "Blob size out of bounds") ^^ + BigNum.to_word64_with env (Blob.lit env Tagged.T "Blob size out of bounds") ^^ Region.load_blob env | OtherPrim ("regionStoreBlob"), [e0; e1; e2] -> @@ -11744,8 +11784,18 @@ and compile_prim_invocation (env : E.t) ae p es at = compile_exp_vanilla env ae e ^^ IC.trap_text env - | OtherPrim ("blobToArray" | "blobToArrayMut"), e -> - const_sr SR.Vanilla (Arr.ofBlob env) + | OtherPrim "principalOfBlob", e -> + const_sr SR.Vanilla (Blob.copy env Tagged.B Tagged.P) + | OtherPrim "blobOfPrincipal", e -> + const_sr SR.Vanilla (Blob.copy env Tagged.P Tagged.B) + | OtherPrim "principalOfActor", e -> + const_sr SR.Vanilla (Blob.copy env Tagged.A Tagged.P) + + | OtherPrim "blobToArray", e -> + const_sr SR.Vanilla (Arr.ofBlob env Tagged.I) + | OtherPrim "blobToArrayMut", e -> + const_sr SR.Vanilla (Arr.ofBlob env Tagged.M) + | OtherPrim ("arrayToBlob" | "arrayMutToBlob"), e -> const_sr SR.Vanilla (Arr.toBlob env) @@ -11830,7 +11880,7 @@ and compile_prim_invocation (env : E.t) ae p es at = SR.Vanilla, compile_exp_as env ae (SR.UnboxedWord64 Type.Nat64) e1 ^^ compile_exp_as env ae SR.Vanilla e2 ^^ - BigNum.to_word64_with env (Blob.lit env "Blob size out of bounds") ^^ + BigNum.to_word64_with env (Blob.lit env Tagged.T "Blob size out of bounds") ^^ StableMemoryInterface.load_blob env | OtherPrim "stableMemoryStoreBlob", [e1; e2] -> @@ -11849,9 +11899,9 @@ and compile_prim_invocation (env : E.t) ae p es at = StableMemoryInterface.grow env | OtherPrim "stableVarQuery", [] -> - SR.UnboxedTuple 2, + SR.Vanilla, IC.get_self_reference env ^^ - Blob.lit env Type.(motoko_stable_var_info_fld.lab) + IC.actor_public_field env Type.(motoko_stable_var_info_fld.lab) (* Other prims, binary*) | OtherPrim "Array.init", [_;_] -> @@ -11903,7 +11953,8 @@ and compile_prim_invocation (env : E.t) ae p es at = compile_unboxed_const 29L ^^ compile_comparison I64Op.LeU ^^ E.else_trap_with env "blob too long for actor principal" ^^ - get_blob + get_blob ^^ + Blob.copy env Tagged.B Tagged.A | SelfRef _, [] -> SR.Vanilla, IC.get_self_reference env @@ -12314,10 +12365,12 @@ and compile_lit_pat env l = | CharLit _ -> compile_lit_as env SR.Vanilla l ^^ compile_eq env Type.(Prim Char) - | TextLit t - | BlobLit t -> + | TextLit t -> compile_lit_as env SR.Vanilla l ^^ Text.compare env Operator.EqOp + | BlobLit t -> + compile_lit_as env SR.Vanilla l ^^ + Blob.compare env (Some Operator.EqOp) | FloatLit _ -> todo_trap env "compile_lit_pat" (Arrange_ir.lit l) @@ -12615,16 +12668,19 @@ and compile_const_exp env pre_ae exp : Const.v * (E.t -> VarEnv.t -> unit) = | PrimE (ProjPrim i, [e]) -> let (object_ct, fill) = compile_const_exp env pre_ae e in let cs = match object_ct with - | Const.Array cs -> cs + | Const.Tuple cs -> cs | _ -> fatal "compile_const_exp/ProjE: not a static tuple" in (List.nth cs i, fill) | LitE l -> Const.(Lit (const_lit_of_lit l)), (fun _ _ -> ()) | PrimE (TupPrim, []) -> Const.Unit, (fun _ _ -> ()) - | PrimE (ArrayPrim (Const, _), es) - | PrimE (TupPrim, es) -> + | PrimE (ArrayPrim (Const, _), es) -> let (cs, fills) = List.split (List.map (compile_const_exp env pre_ae) es) in (Const.Array cs), (fun env ae -> List.iter (fun fill -> fill env ae) fills) + | PrimE (TupPrim, es) -> + let (cs, fills) = List.split (List.map (compile_const_exp env pre_ae) es) in + (Const.Tuple cs), + (fun env ae -> List.iter (fun fill -> fill env ae) fills) | PrimE (TagPrim i, [e]) -> let (arg_ct, fill) = compile_const_exp env pre_ae e in (Const.Tag (i, arg_ct)), @@ -12669,7 +12725,7 @@ and destruct_const_pat ae pat const : VarEnv.t option = match pat.it with if l = None then destruct_const_pat ae p2 const else l | TupP ps -> - let cs = match const with Const.Array cs -> cs | Const.Unit -> [] | _ -> assert false in + let cs = match const with Const.Tuple cs -> cs | Const.Unit -> [] | _ -> assert false in let go ae p c = match ae with | Some ae -> destruct_const_pat ae p c | _ -> None in diff --git a/src/mo_values/prim.ml b/src/mo_values/prim.ml index 64ea0bad606..21f62d1cb4c 100644 --- a/src/mo_values/prim.ml +++ b/src/mo_values/prim.ml @@ -293,6 +293,12 @@ let prim trap = in go (fun xs -> xs) k 0 | _ -> assert false ) + + + | "blobOfPrincipal" -> fun _ v k -> k v + | "principalOfBlob" -> fun _ v k -> k v + | "principalOfActor" -> fun _ v k -> k v + | "blobToArray" -> fun _ v k -> k (Array (Array.of_seq (Seq.map (fun c -> Nat8 (Nat8.of_int (Char.code c)) diff --git a/src/prelude/internals.mo b/src/prelude/internals.mo index 558f0cea57c..a0ef9df6435 100644 --- a/src/prelude/internals.mo +++ b/src/prelude/internals.mo @@ -441,17 +441,17 @@ func @install_actor_helper( (#install, principal1) }; case (#reinstall actor1) { - (#reinstall, (prim "cast" : (actor {}) -> Principal) actor1) + (#reinstall, (prim "principalOfActor" : (actor {}) -> Principal) actor1) }; case (#upgrade actor2) { let upgradeOptions = { wasm_memory_persistence = ?(#Keep); }; - ((#upgrade (?upgradeOptions)), (prim "cast" : (actor {}) -> Principal) actor2) + ((#upgrade (?upgradeOptions)), (prim "principalOfActor" : (actor {}) -> Principal) actor2) }; case (#upgrade_with_persistence { wasm_memory_persistence; canister } ) { let upgradeOptions = { wasm_memory_persistence = ?wasm_memory_persistence }; - ((#upgrade (?upgradeOptions)), (prim "cast" : (actor {}) -> Principal) canister) + ((#upgrade (?upgradeOptions)), (prim "principalOfActor" : (actor {}) -> Principal) canister) }; }; await @ic00.install_code { diff --git a/src/prelude/prim.mo b/src/prelude/prim.mo index 8dc708af2ad..cc589e5da84 100644 --- a/src/prelude/prim.mo +++ b/src/prelude/prim.mo @@ -310,15 +310,16 @@ func time() : Nat64 = (prim "time" : () -> Nat64)(); // Principal -func blobOfPrincipal(id : Principal) : Blob = (prim "cast" : Principal -> Blob) id; +func blobOfPrincipal(id : Principal) : Blob = (prim "blobOfPrincipal" : Principal -> Blob) id; func principalOfBlob(act : Blob) : Principal { + // TODO: better: check size in prim "principalOfBob" instead if (act.size() > 29) { trap("blob too long for principal"); }; - (prim "cast" : Blob -> Principal) act; + (prim "principalOfBlob" : Blob -> Principal) act; }; -func principalOfActor(act : actor {}) : Principal = (prim "cast" : (actor {}) -> Principal) act; +func principalOfActor(act : actor {}) : Principal = (prim "principalOfActor" : (actor {}) -> Principal) act; func isController(p : Principal) : Bool = (prim "is_controller" : Principal -> Bool) p; func canisterVersion() : Nat64 = (prim "canister_version" : () -> Nat64)(); diff --git a/test/bench/ok/bignum.drun-run.ok b/test/bench/ok/bignum.drun-run.ok index bcd3c08cc93..999dcf88666 100644 --- a/test/bench/ok/bignum.drun-run.ok +++ b/test/bench/ok/bignum.drun-run.ok @@ -2,5 +2,5 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a000000000000000001 ingress Completed: Reply: 0x4449444c0000 debug.print: {cycles = 3_094_308; size = +25_528} ingress Completed: Reply: 0x4449444c0000 -debug.print: {cycles = 51_911_770; size = +830_464} +debug.print: {cycles = 51_911_784; size = +830_464} ingress Completed: Reply: 0x4449444c0000 diff --git a/test/bench/ok/candid-subtype-cost.drun-run.ok b/test/bench/ok/candid-subtype-cost.drun-run.ok index c19e4c4e50b..f018dca7e67 100644 --- a/test/bench/ok/candid-subtype-cost.drun-run.ok +++ b/test/bench/ok/candid-subtype-cost.drun-run.ok @@ -1,4 +1,4 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: {cycles = 1_102_516; heap_bytes = +33_872} +debug.print: {cycles = 1_103_543; heap_bytes = +33_872} ingress Completed: Reply: 0x4449444c0000 diff --git a/test/bench/ok/heap-32.drun-run.ok b/test/bench/ok/heap-32.drun-run.ok index 067c79f8af7..595bac1a944 100644 --- a/test/bench/ok/heap-32.drun-run.ok +++ b/test/bench/ok/heap-32.drun-run.ok @@ -1,5 +1,5 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: (50_227, +75_320_504, 1_272_634_370) -debug.print: (50_070, +86_221_288, 1_390_376_355) +debug.print: (50_227, +75_320_504, 1_272_634_374) +debug.print: (50_070, +86_221_288, 1_390_376_359) ingress Completed: Reply: 0x4449444c0000 diff --git a/test/bench/ok/heap-64.drun-run.ok b/test/bench/ok/heap-64.drun-run.ok index 4ffad0a3872..f67af0b38d0 100644 --- a/test/bench/ok/heap-64.drun-run.ok +++ b/test/bench/ok/heap-64.drun-run.ok @@ -1,5 +1,5 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: (49_965, +88_051_888, 1_341_531_415) -debug.print: (49_806, +86_432_120, 1_304_067_079) +debug.print: (49_965, +88_051_888, 1_341_531_419) +debug.print: (49_806, +86_432_120, 1_304_067_083) ingress Completed: Reply: 0x4449444c0000 diff --git a/test/run/refuted-const-float.mo b/test/run/refuted-const-float.mo index 2bd1dad82f3..8e2e839c321 100644 --- a/test/run/refuted-const-float.mo +++ b/test/run/refuted-const-float.mo @@ -1,11 +1,5 @@ // a failing pattern match that can be compiled to a trap let 0.67 = 3.14; -// CHECK: (func $init (type -// CHECK: call $blob_of_principal -// CHECK: i64.const 14 -// CHECK: call $print_ptr -// CHECK-NEXT: unreachable) - //SKIP run-low //SKIP run-ir diff --git a/test/run/refuted-const-option-null.mo b/test/run/refuted-const-option-null.mo index 78df851ae40..ccc65afd6c9 100644 --- a/test/run/refuted-const-option-null.mo +++ b/test/run/refuted-const-option-null.mo @@ -1,11 +1,5 @@ // a failing pattern match that can be compiled to a trap let ?b = null; -// CHECK: (func $init (type -// CHECK: call $blob_of_principal -// CHECK: i64.const 14 -// CHECK: call $print_ptr -// CHECK-NEXT: unreachable) - //SKIP run-low //SKIP run-ir diff --git a/test/run/refuted-const-option.mo b/test/run/refuted-const-option.mo index f3fa08a2440..37e944c1f19 100644 --- a/test/run/refuted-const-option.mo +++ b/test/run/refuted-const-option.mo @@ -1,11 +1,5 @@ // a failing pattern match that can be compiled to a trap let null = ?42; -// CHECK: (func $init (type -// CHECK: call $blob_of_principal -// CHECK: i64.const 14 -// CHECK: call $print_ptr -// CHECK-NEXT: unreachable) - //SKIP run-low //SKIP run-ir diff --git a/test/run/refuted-const-variant.mo b/test/run/refuted-const-variant.mo index 0926852a75d..161d096126c 100644 --- a/test/run/refuted-const-variant.mo +++ b/test/run/refuted-const-variant.mo @@ -1,11 +1,5 @@ // a failing pattern match that can be compiled to a trap let (#const b) = #bummer; -// CHECK: (func $init (type -// CHECK: call $blob_of_principal -// CHECK: i64.const 14 -// CHECK: call $print_ptr -// CHECK-NEXT: unreachable) - //SKIP run-low //SKIP run-ir