1use std::any::{Any, TypeId};
14use std::cell::Cell;
15
16use anyhow::{Result, anyhow, ensure};
17
18use crate::canonical_operation::OperationPlan;
19use crate::container_execution::{ContainerPlan, ResultPlan, StackFramePlan, stack_position};
20use crate::host_value_bridge::HostValuePlan;
21use crate::owned_value::OwnedValue;
22use crate::{
23 AssociatedType, CNil, CStackList, CStackListHeadLimit, CStackListHeadPadded, IntoCStackList,
24 Leaf, List, RawSegment, RawStack, ReverseList, ValueKind, ValueRepr, ValueType,
25};
26
27thread_local! {
28 static ARGUMENTS: Cell<*const *const u8> = const { Cell::new(std::ptr::null()) };
30}
31
32struct ArgumentGuard(*const *const u8);
34
35impl ArgumentGuard {
36 fn install(arguments: &[*const u8]) -> Self {
38 Self(ARGUMENTS.with(|slot| slot.replace(arguments.as_ptr())))
39 }
40}
41
42impl Drop for ArgumentGuard {
43 fn drop(&mut self) {
45 ARGUMENTS.with(|slot| slot.set(self.0));
46 }
47}
48
49#[derive(Clone, Debug)]
51pub struct StackInfo {
52 pub padding: bool,
54 pub value_type: ValueType,
56}
57
58fn descriptor_name(ty: &ValueType) -> String {
62 match ty.kind() {
63 ValueKind::Array(element) => format!("[{}]", descriptor_name(element)),
64 ValueKind::Tuple(fields) => format!(
65 "({})",
66 fields
67 .iter()
68 .map(|field| descriptor_name(&field.value_type))
69 .collect::<Vec<_>>()
70 .join(", ")
71 ),
72 _ => ty.type_name().to_owned(),
73 }
74}
75
76pub trait ToTypeIdList: List {
78 fn to_stack_info_list() -> Vec<StackInfo>;
82}
83
84impl ToTypeIdList for CNil<()> {
85 fn to_stack_info_list() -> Vec<StackInfo> {
87 Vec::new()
88 }
89}
90
91impl<H: 'static, T: ToTypeIdList + 'static + CStackListHeadLimit> ToTypeIdList
92 for CStackList<H, T>
93{
94 fn to_stack_info_list() -> Vec<StackInfo> {
98 let mut result = T::to_stack_info_list();
99 result.push(StackInfo {
100 padding: Self::HEAD_PADDED,
101 value_type: ValueType::leaf::<H>(),
102 });
103 result
104 }
105}
106
107pub struct DynSegment {
112 pub(crate) segment: RawSegment,
113 pub(crate) argument_ids: Vec<TypeId>,
114 pub(crate) stack_ids: Vec<StackInfo>,
115 pub(crate) base_stack_index: usize,
116 pub(crate) argument_reads: Vec<(usize, ValueType)>,
117 stack_ends: Vec<usize>,
119}
120
121impl DynSegment {
122 pub fn new<Args: IntoCStackList>() -> Self
132 where
133 ReverseList<Args::Output>: ToTypeIdList,
134 {
135 let stack_ids = ReverseList::<Args::Output>::to_stack_info_list();
136 let mut segment = RawSegment::new();
137 segment.update_base_alignment(1);
138 for info in &stack_ids {
139 info.value_type
140 .validate_canonical()
141 .expect("preloaded dynamic arguments must be canonical or ordinary opaque leaves");
142 segment.update_base_alignment(info.value_type.align());
143 }
144 segment.set_frame(0, stack_ids.len());
145 let mut stack_ends = Vec::with_capacity(stack_ids.len() + 1);
146 stack_ends.push(0);
147 for info in &stack_ids {
148 stack_ends.push(
149 stack_position(*stack_ends.last().unwrap(), info.value_type.layout())
150 .expect("validated preloaded stack")
151 .1,
152 );
153 }
154 Self {
155 segment,
156 argument_ids: stack_ids
157 .iter()
158 .map(|info| info.value_type.type_id())
159 .collect(),
160 stack_ids,
161 base_stack_index: 0,
162 argument_reads: Vec::new(),
163 stack_ends,
164 }
165 }
166
167 pub fn new_fragment(&self) -> Self {
174 debug_assert!(!self.stack_ids.is_empty());
175 let mut result = Self::new::<()>();
176 result.base_stack_index = self.stack_offset_before_top(1).expect("condition present");
177 result.stack_ends[0] = result.base_stack_index;
178 result.segment.set_frame(result.base_stack_index, 0);
179 result
180 }
181
182 pub fn current_stack_offset(&self) -> usize {
189 self.offset_after(self.stack_ids.len())
190 }
191
192 pub fn stack_offset_before_top(&self, n: usize) -> Result<usize> {
202 let count = self
203 .stack_ids
204 .len()
205 .checked_sub(n)
206 .ok_or_else(|| anyhow!("not enough operands"))?;
207 Ok(self.offset_after(count))
208 }
209
210 fn offset_after(&self, count: usize) -> usize {
212 self.stack_ends[count]
213 }
214
215 fn operation(&self, arity: usize, result: ValueType) -> Result<OperationPlan> {
221 let first = self
222 .stack_ids
223 .len()
224 .checked_sub(arity)
225 .ok_or_else(|| anyhow!("not enough operands"))?;
226 let operands: Vec<_> = self.stack_ids[first..]
227 .iter()
228 .map(|info| info.value_type.clone())
229 .collect();
230 OperationPlan::at(
231 self.base_stack_index,
232 self.offset_after(first),
233 first,
234 &operands,
235 result,
236 )
237 }
238
239 fn replace(&mut self, arity: usize, result: ValueType) {
243 self.stack_ids.truncate(self.stack_ids.len() - arity);
244 self.stack_ends.truncate(self.stack_ids.len() + 1);
245 let before = self.current_stack_offset();
246 let (offset, end) = stack_position(before, result.layout()).expect("validated result");
247 self.segment.update_base_alignment(result.align());
248 self.stack_ids.push(StackInfo {
249 padding: offset != before,
250 value_type: result,
251 });
252 self.stack_ends.push(end);
253 }
254
255 pub fn peek_stack_infos(&self, n: usize) -> &[StackInfo] {
263 &self.stack_ids[self.stack_ids.len().saturating_sub(n)..]
264 }
265
266 pub fn peek_output_type_id(&self) -> Option<TypeId> {
273 Some(self.stack_ids.last()?.value_type.type_id())
274 }
275
276 pub fn peek_tuple_arity(&self) -> Option<usize> {
283 Some(self.stack_ids.last()?.value_type.tuple_elements()?.len())
284 }
285
286 pub fn op0<R: 'static, F: Fn() -> R + 'static>(&mut self, f: F) {
299 self.op0_repr::<Leaf<R>, _>(f)
300 .expect("opaque result fits raw frame");
301 }
302
303 pub fn op0r<R: 'static, F: Fn() -> Result<R> + 'static>(&mut self, f: F) {
314 self.op0r_repr::<Leaf<R>, _>(f)
315 .expect("opaque result fits raw frame");
316 }
317
318 pub fn op0_repr<R: ValueRepr, F: Fn() -> R::Value + 'static>(&mut self, f: F) -> Result<()> {
330 self.op0r_repr::<R, _>(move || Ok(f()))
331 }
332
333 pub fn op0r_repr<R: ValueRepr, F: Fn() -> Result<R::Value> + 'static>(
345 &mut self,
346 f: F,
347 ) -> Result<()> {
348 let result = R::value_type();
349 self.operation(0, result.clone())?
350 .emit0::<R, _>(&mut self.segment, f)?;
351 self.replace(0, result);
352 Ok(())
353 }
354
355 pub fn op1<A: 'static, R: 'static, F: Fn(A) -> R + 'static>(&mut self, f: F) -> Result<()> {
367 self.op1_repr::<Leaf<A>, Leaf<R>, _>(f)
368 }
369
370 pub fn op1r<A: 'static, R: 'static, F: Fn(A) -> Result<R> + 'static>(
382 &mut self,
383 f: F,
384 ) -> Result<()> {
385 self.op1r_repr::<Leaf<A>, Leaf<R>, _>(f)
386 }
387
388 pub fn op1_repr<A: ValueRepr, R: ValueRepr, F: Fn(A::Value) -> R::Value + 'static>(
401 &mut self,
402 f: F,
403 ) -> Result<()> {
404 self.op1r_repr::<A, R, _>(move |a| Ok(f(a)))
405 }
406
407 pub fn op1r_repr<A: ValueRepr, R: ValueRepr, F: Fn(A::Value) -> Result<R::Value> + 'static>(
420 &mut self,
421 f: F,
422 ) -> Result<()> {
423 let result = R::value_type();
424 self.operation(1, result.clone())?
425 .emit1::<A, R, _>(&mut self.segment, f)?;
426 self.replace(1, result);
427 Ok(())
428 }
429
430 pub fn op2<A: 'static, B: 'static, R: 'static, F: Fn(A, B) -> R + 'static>(
442 &mut self,
443 f: F,
444 ) -> Result<()> {
445 self.op2_repr::<Leaf<A>, Leaf<B>, Leaf<R>, _>(f)
446 }
447
448 pub fn op2r<A: 'static, B: 'static, R: 'static, F: Fn(A, B) -> Result<R> + 'static>(
460 &mut self,
461 f: F,
462 ) -> Result<()> {
463 self.op2r_repr::<Leaf<A>, Leaf<B>, Leaf<R>, _>(f)
464 }
465
466 pub fn op2_repr<
475 A: ValueRepr,
476 B: ValueRepr,
477 R: ValueRepr,
478 F: Fn(A::Value, B::Value) -> R::Value + 'static,
479 >(
480 &mut self,
481 f: F,
482 ) -> Result<()> {
483 self.op2r_repr::<A, B, R, _>(move |a, b| Ok(f(a, b)))
484 }
485
486 pub fn op2r_repr<
495 A: ValueRepr,
496 B: ValueRepr,
497 R: ValueRepr,
498 F: Fn(A::Value, B::Value) -> Result<R::Value> + 'static,
499 >(
500 &mut self,
501 f: F,
502 ) -> Result<()> {
503 let result = R::value_type();
504 self.operation(2, result.clone())?
505 .emit2::<A, B, R, _>(&mut self.segment, f)?;
506 self.replace(2, result);
507 Ok(())
508 }
509
510 pub fn op3<A: 'static, B: 'static, C: 'static, R: 'static, F: Fn(A, B, C) -> R + 'static>(
519 &mut self,
520 f: F,
521 ) -> Result<()> {
522 self.op3_repr::<Leaf<A>, Leaf<B>, Leaf<C>, Leaf<R>, _>(f)
523 }
524
525 pub fn op3r<
534 A: 'static,
535 B: 'static,
536 C: 'static,
537 R: 'static,
538 F: Fn(A, B, C) -> Result<R> + 'static,
539 >(
540 &mut self,
541 f: F,
542 ) -> Result<()> {
543 self.op3r_repr::<Leaf<A>, Leaf<B>, Leaf<C>, Leaf<R>, _>(f)
544 }
545
546 pub fn op3_repr<
555 A: ValueRepr,
556 B: ValueRepr,
557 C: ValueRepr,
558 R: ValueRepr,
559 F: Fn(A::Value, B::Value, C::Value) -> R::Value + 'static,
560 >(
561 &mut self,
562 f: F,
563 ) -> Result<()> {
564 self.op3r_repr::<A, B, C, R, _>(move |a, b, c| Ok(f(a, b, c)))
565 }
566
567 pub fn op3r_repr<
576 A: ValueRepr,
577 B: ValueRepr,
578 C: ValueRepr,
579 R: ValueRepr,
580 F: Fn(A::Value, B::Value, C::Value) -> Result<R::Value> + 'static,
581 >(
582 &mut self,
583 f: F,
584 ) -> Result<()> {
585 let result = R::value_type();
586 self.operation(3, result.clone())?
587 .emit3::<A, B, C, R, _>(&mut self.segment, f)?;
588 self.replace(3, result);
589 Ok(())
590 }
591
592 pub fn op1_vec<E: ValueRepr, R: ValueRepr, F: Fn(Vec<E::Value>) -> R::Value + 'static>(
601 &mut self,
602 f: F,
603 ) -> Result<()> {
604 self.op1r_vec::<E, R, _>(move |a| Ok(f(a)))
605 }
606
607 pub fn op1r_vec<
616 E: ValueRepr,
617 R: ValueRepr,
618 F: Fn(Vec<E::Value>) -> Result<R::Value> + 'static,
619 >(
620 &mut self,
621 f: F,
622 ) -> Result<()> {
623 let result = R::value_type();
624 self.operation(1, result.clone())?
625 .emit_vec::<E, R, _>(&mut self.segment, f)?;
626 self.replace(1, result);
627 Ok(())
628 }
629
630 pub fn op1_slice<E: ValueRepr, R: ValueRepr, F: Fn(&[E::Value]) -> R::Value + 'static>(
640 &mut self,
641 f: F,
642 ) -> Result<()> {
643 self.op1r_slice::<E, R, _>(move |a| Ok(f(a)))
644 }
645
646 pub fn op1r_slice<
655 E: ValueRepr,
656 R: ValueRepr,
657 F: Fn(&[E::Value]) -> Result<R::Value> + 'static,
658 >(
659 &mut self,
660 f: F,
661 ) -> Result<()> {
662 let result = R::value_type();
663 self.operation(1, result.clone())?
664 .emit_slice::<E, R, _>(&mut self.segment, f)?;
665 self.replace(1, result);
666 Ok(())
667 }
668
669 pub fn just<T: Clone + 'static>(&mut self, value: T) {
679 self.op0(move || value.clone());
680 self.replace(1, ValueType::cloneable_leaf::<T>());
681 }
682
683 pub fn just_external(&mut self, value_type: ValueType, value: Box<dyn Any>) -> Result<()> {
702 value_type.validate_canonical()?;
703 value_type.validate_clone()?;
704 let owner = HostValuePlan::new(value_type.clone())?.import_box(&mut Some(value))?;
705 let result = ResultPlan::new(self.current_stack_offset(), value_type.clone())?;
706 self.segment.update_base_alignment(value_type.align());
707 self.segment.raw0_(move |stack| {
708 unsafe {
711 result.clone_from(stack, owner.as_ptr());
712 }
713 Ok(())
714 });
715 self.replace(0, value_type);
716 Ok(())
717 }
718
719 pub fn push_arg<T: Clone + 'static>(&mut self, index: usize) {
728 self.push_argument(index, ValueType::cloneable_leaf::<T>())
729 .expect("cloneable leaf");
730 }
731
732 pub fn push_argument(&mut self, index: usize, value_type: ValueType) -> Result<()> {
746 value_type.validate_canonical()?;
747 ensure!(
748 value_type.can_clone(),
749 "argument clone capability unavailable"
750 );
751 let result = ResultPlan::new(self.current_stack_offset(), value_type.clone())?;
752 self.segment.update_base_alignment(value_type.align());
753 self.segment.raw0_(move |stack| {
754 unsafe {
757 let source = ARGUMENTS.with(|slot| *slot.get().add(index));
758 result.clone_from(stack, source);
759 }
760 Ok(())
761 });
762 self.argument_reads.push((index, value_type.clone()));
763 self.replace(0, value_type);
764 Ok(())
765 }
766
767 pub fn make_tuple(&mut self, n: usize, ambient_start: usize) -> Result<()> {
779 ensure!(
780 self.stack_offset_before_top(n)? == ambient_start,
781 "tuple ambient origin mismatch"
782 );
783 let fields = self
784 .peek_stack_infos(n)
785 .iter()
786 .map(|info| AssociatedType {
787 offset: 0,
788 value_type: info.value_type.clone(),
789 })
790 .collect();
791 let value_type = ValueType::tuple(fields)?;
792 let frame = StackFramePlan::new(ambient_start, value_type.clone())?;
793 let result = ResultPlan::new(ambient_start, value_type.clone())?;
794 self.segment.update_base_alignment(value_type.align());
795 self.segment.raw0_(move |stack| {
796 let mut owner = unsafe { frame.take(stack) };
798 unsafe { result.push(stack, &mut owner) };
799 Ok(())
800 });
801 self.replace(n, value_type);
802 Ok(())
803 }
804
805 pub fn make_array(&mut self, n: usize, ambient_start: usize) -> Result<()> {
818 ensure!(
819 n != 0,
820 "empty array requires an explicit complete element type"
821 );
822 self.stack_offset_before_top(n)?;
823 let element = self.peek_stack_infos(n)[0].value_type.clone();
824 self.make_typed_array(n, ambient_start, element)
825 }
826
827 pub fn make_typed_array(
840 &mut self,
841 n: usize,
842 ambient_start: usize,
843 element: ValueType,
844 ) -> Result<()> {
845 element.validate_canonical()?;
846 ensure!(
847 self.stack_offset_before_top(n)? == ambient_start,
848 "array ambient origin mismatch"
849 );
850 for (index, info) in self.peek_stack_infos(n).iter().enumerate() {
851 ensure!(
852 info.value_type.same_shape(&element),
853 "array element {index} has type {}, expected {}",
854 descriptor_name(&info.value_type),
855 descriptor_name(&element)
856 );
857 }
858 let container = ContainerPlan::array(element.clone(), n)?;
859 let result_type = ValueType::array(element.clone());
860 let result = ResultPlan::new(ambient_start, result_type.clone())?;
861 let mut end = ambient_start;
862 let offsets = (0..n)
863 .map(|_| {
864 let (offset, next) = stack_position(end, element.layout())?;
865 end = next;
866 Ok(offset)
867 })
868 .collect::<Result<Vec<_>>>()?;
869 self.segment.update_base_alignment(result_type.align());
870 self.segment.raw0_(move |stack| {
871 let mut owner = unsafe {
873 container.construct(|index, destination| {
874 stack.read_at(offsets[index], |source| {
875 std::ptr::copy_nonoverlapping(source, destination, element.size())
876 });
877 Ok(())
878 })?
879 };
880 unsafe {
881 stack.release_top(n);
882 stack.truncate_to(ambient_start, false);
883 result.push(stack, &mut owner);
884 }
885 Ok(())
886 });
887 self.replace(n, result_type);
888 Ok(())
889 }
890
891 pub fn tuple_index(&mut self, index: usize) -> Result<()> {
904 let ty = self
905 .stack_ids
906 .last()
907 .ok_or_else(|| anyhow!("missing tuple"))?
908 .value_type
909 .clone();
910 let fields = ty
911 .tuple_elements()
912 .ok_or_else(|| anyhow!("tuple required"))?;
913 let field = fields
914 .get(index)
915 .ok_or_else(|| anyhow!("tuple index out of range"))?
916 .value_type
917 .clone();
918 let ambient = self.stack_offset_before_top(1)?;
919 let input = ResultPlan::new(ambient, ty)?;
920 let output = ResultPlan::new(ambient, field.clone())?;
921 self.segment.raw0_(move |stack| {
922 let owner = unsafe { input.take(stack) };
923 let mut selected = unsafe { owner.take_field(index) };
924 unsafe { output.push(stack, &mut selected) };
925 Ok(())
926 });
927 self.replace(1, field);
928 Ok(())
929 }
930
931 pub fn array_index<I: TryInto<usize> + 'static>(&mut self) -> Result<()> {
949 let first = self
950 .stack_ids
951 .len()
952 .checked_sub(2)
953 .ok_or_else(|| anyhow!("array index requires two operands"))?;
954 self.stack_ids[first + 1]
955 .value_type
956 .validate_opaque_leaf(TypeId::of::<I>())?;
957 let array_type = self.stack_ids[first].value_type.clone();
958 let element = array_type
959 .array_element()
960 .ok_or_else(|| anyhow!("canonical array required"))?
961 .clone();
962 let ambient = self.offset_after(first);
963 let index = ResultPlan::new(self.offset_after(first + 1), ValueType::leaf::<I>())?;
964 let array = ResultPlan::new(ambient, array_type)?;
965 let output = ResultPlan::new(ambient, element.clone())?;
966 self.segment.update_base_alignment(element.align());
967 self.segment.raw0_(move |stack| {
968 let native = unsafe { index.take_typed::<I>(stack) };
970 let array_owner = unsafe { array.take(stack) };
971 let selected = native
972 .try_into()
973 .map_err(|_| anyhow!("array index is negative or unrepresentable"))
974 .and_then(|index| unsafe { array_owner.take_array_element(index) });
975 match selected {
976 Ok(mut value) => {
977 unsafe { output.push(stack, &mut value) };
978 Ok(())
979 }
980 Err(error) => Err(error),
981 }
982 });
983 self.replace(2, element);
984 Ok(())
985 }
986
987 pub fn drop_value(&mut self) -> Result<()> {
1002 let first = self
1003 .stack_ids
1004 .len()
1005 .checked_sub(1)
1006 .ok_or_else(|| anyhow!("missing value"))?;
1007 self.segment.raw0_(|stack| {
1008 stack.drop_top();
1009 Ok(())
1010 });
1011 self.stack_ids.truncate(first);
1012 self.stack_ends.truncate(first + 1);
1013 Ok(())
1014 }
1015
1016 pub fn relabel_repr<R: ValueRepr>(&mut self) -> Result<()> {
1028 self.stack_ids
1029 .last()
1030 .ok_or_else(|| anyhow!("missing value"))?
1031 .value_type
1032 .validate::<R>()
1033 }
1034
1035 pub fn annotate_top(&mut self, value_type: ValueType) -> Result<()> {
1052 let info = self
1053 .stack_ids
1054 .last_mut()
1055 .ok_or_else(|| anyhow!("missing value"))?;
1056 ensure!(
1057 info.value_type.same_shape(&value_type),
1058 "top value annotation representation mismatch"
1059 );
1060 info.value_type = value_type;
1061 Ok(())
1062 }
1063
1064 pub fn pop_tuple_as<R: ValueRepr>(&mut self) -> Result<()> {
1079 let info = self
1080 .stack_ids
1081 .last_mut()
1082 .ok_or_else(|| anyhow!("missing tuple"))?;
1083 ensure!(
1084 info.value_type.tuple_elements().is_some(),
1085 "canonical tuple required"
1086 );
1087 info.value_type.validate::<R>()?;
1088 info.value_type = ValueType::leaf::<R::Value>();
1089 Ok(())
1090 }
1091
1092 pub fn push_tuple<R: ValueRepr>(&mut self) -> Result<()> {
1106 let value_type = R::value_type();
1107 ensure!(
1108 value_type.tuple_elements().is_some(),
1109 "canonical tuple witness required"
1110 );
1111 let info = self
1112 .stack_ids
1113 .last_mut()
1114 .ok_or_else(|| anyhow!("missing concrete list"))?;
1115 info.value_type
1116 .validate_opaque_leaf(TypeId::of::<R::Value>())?;
1117 debug_assert_eq!(info.value_type.layout(), value_type.layout());
1118 info.value_type = value_type;
1119 Ok(())
1120 }
1121
1122 pub(crate) fn validate_independent_body(&self) -> Result<()> {
1127 ensure!(
1128 self.argument_ids.is_empty(),
1129 "independent body must not require preloaded arguments"
1130 );
1131 ensure!(
1132 self.base_stack_index == 0,
1133 "independent body must not require a nonzero ambient stack origin"
1134 );
1135 Ok(())
1136 }
1137
1138 pub(crate) unsafe fn execute_raw(&mut self, arguments: &[*const u8]) -> Result<RawStack> {
1147 let _guard = ArgumentGuard::install(arguments);
1148 let mut output = RawStack::with_base_alignment(self.segment.base_alignment());
1149 unsafe { self.segment.call0_stack(&mut output)? };
1150 Ok(output)
1151 }
1152
1153 pub fn call_dyn<R: 'static>(&mut self, inputs: &[&dyn Any]) -> Result<R> {
1168 let result = self.result_type()?;
1169 match result.kind() {
1170 ValueKind::Tuple(fields)
1171 if fields.is_empty() && TypeId::of::<R>() == TypeId::of::<()>() =>
1172 {
1173 drop(self.call_dyn_value(inputs)?);
1174 let unit: Box<dyn Any> = Box::new(());
1175 return Ok(*unit.downcast::<R>().expect("checked host unit identity"));
1176 }
1177 ValueKind::Array(_) | ValueKind::Tuple(_) => {
1178 ensure!(
1179 TypeId::of::<R>() == TypeId::of::<OwnedValue>(),
1180 "canonical result owner required"
1181 );
1182 let owner: Box<dyn Any> = Box::new(self.call_dyn_value(inputs)?);
1183 return Ok(*owner
1184 .downcast::<R>()
1185 .map_err(|_| anyhow!("canonical result owner required"))?);
1186 }
1187 ValueKind::Callable { .. } => {
1188 ensure!(
1189 TypeId::of::<R>() == TypeId::of::<crate::DynClosure>(),
1190 "external callable result type mismatch"
1191 );
1192 }
1193 ValueKind::Leaf(_) => result.validate_opaque_leaf(TypeId::of::<R>())?,
1194 }
1195 let plan = ResultPlan::new(0, self.result_type()?.clone())?;
1196 let mut output = self.call_external(inputs)?;
1197 Ok(unsafe { plan.take_typed::<R>(&mut output) })
1198 }
1199
1200 pub fn call_dyn_value(&mut self, inputs: &[&dyn Any]) -> Result<OwnedValue> {
1214 let result = ResultPlan::new(0, self.result_type()?.clone())?;
1215 let mut output = self.call_external(inputs)?;
1216 Ok(unsafe { result.take(&mut output) })
1217 }
1218
1219 fn call_external(&mut self, inputs: &[&dyn Any]) -> Result<RawStack> {
1230 self.validate_independent_body()?;
1231 let mut first_reads: Vec<Option<&ValueType>> = vec![None; inputs.len()];
1232 for (index, ty) in &self.argument_reads {
1233 ensure!(*index < inputs.len(), "missing argument {index}");
1234 if let Some(first) = first_reads[*index] {
1235 ensure!(
1236 ty.same_shape(first),
1237 "inconsistent argument {index} representation"
1238 );
1239 } else {
1240 first_reads[*index] = Some(ty);
1241 }
1242 }
1243 let mut pointers = vec![std::ptr::null(); inputs.len()];
1244 for (index, ty) in first_reads.into_iter().enumerate() {
1245 if let Some(ty) = ty {
1246 pointers[index] = HostValuePlan::new(ty.clone())?.validate_any(inputs[index])?;
1247 }
1248 }
1249 unsafe { self.execute_raw(&pointers) }
1250 }
1251
1252 pub(crate) fn result_type(&self) -> Result<&ValueType> {
1257 ensure!(
1258 self.stack_ids.len() == 1,
1259 "body must produce exactly one value"
1260 );
1261 Ok(&self.stack_ids[0].value_type)
1262 }
1263
1264 pub fn call0<R: 'static>(&mut self) -> Result<R> {
1273 self.call_dyn(&[])
1274 }
1275
1276 pub fn call1<A: 'static, R: 'static>(&mut self, arg: A) -> Result<R> {
1289 ensure!(
1290 self.argument_ids == [TypeId::of::<A>()],
1291 "argument type mismatch"
1292 );
1293 ensure!(
1294 self.argument_reads.is_empty(),
1295 "preloaded call cannot use positional argument reads"
1296 );
1297 self.result_type()?
1298 .validate_opaque_leaf(TypeId::of::<R>())?;
1299 unsafe { self.segment.call1(arg) }
1300 }
1301
1302 pub fn prepare<A: ValueRepr, R: ValueRepr>(self) -> Result<crate::PreparedSegment<A, R>> {
1312 crate::PreparedSegment::new(self)
1313 }
1314
1315 pub fn push_closure(&mut self, closure: crate::DynClosure) -> Result<()> {
1324 let signature = closure
1325 .signature()
1326 .ok_or_else(|| anyhow!("closure signature metadata is unavailable"))?;
1327 closure.validate_body_signature()?;
1328 let ty = ValueType::callable(signature.parameters.clone(), signature.result.clone());
1329 let closure = closure.bind_canonical()?;
1331 self.segment.update_base_alignment(ty.align());
1332 self.segment.raw0_(move |stack| {
1333 stack.push(closure.clone());
1334 Ok(())
1335 });
1336 self.replace(0, ty);
1337 Ok(())
1338 }
1339
1340 pub fn invoke_closure(&mut self, arity: usize) -> Result<()> {
1360 ensure!(
1361 self.argument_ids.is_empty(),
1362 "closure invocation requires a positional body"
1363 );
1364 let first = self
1365 .stack_ids
1366 .len()
1367 .checked_sub(arity + 1)
1368 .ok_or_else(|| anyhow!("missing callee"))?;
1369 let callee_type = self.stack_ids[first].value_type.clone();
1370 let ValueKind::Callable { parameters, result } = callee_type.kind() else {
1371 anyhow::bail!("callable signature required")
1372 };
1373 ensure!(parameters.len() == arity, "closure arity mismatch");
1374 ensure!(
1375 parameters
1376 .iter()
1377 .zip(&self.stack_ids[first + 1..])
1378 .all(|(expected, actual)| expected.same_shape(&actual.value_type)),
1379 "closure argument type mismatch"
1380 );
1381 let result_type = result.as_ref().clone();
1382 let ambient = self.offset_after(first);
1383 let argument_type = ValueType::tuple(
1384 parameters
1385 .iter()
1386 .map(|value_type| AssociatedType {
1387 offset: 0,
1388 value_type: value_type.clone(),
1389 })
1390 .collect(),
1391 )?;
1392 let arguments_start = self.offset_after(first + 1);
1393 let frame = StackFramePlan::new(arguments_start, argument_type.clone())?;
1394 let callee = ResultPlan::new(ambient, callee_type.clone())?;
1395 let destination = ResultPlan::new(ambient, result_type.clone())?;
1396 self.segment.update_base_alignment(result_type.align());
1397 let output_type = result_type.clone();
1398 self.segment.raw0_(move |stack| {
1399 let arguments = unsafe { frame.take(stack) };
1401 let closure = unsafe { callee.take_typed::<crate::DynClosure>(stack) };
1403 let allocation = crate::owned_value::Allocation::new(output_type.layout());
1406 let evaluated = unsafe {
1407 closure.call_frame_trusted(arguments.as_ptr(), allocation.pointer.as_ptr())
1408 };
1409 match evaluated {
1410 Ok(()) => {
1411 let mut result = unsafe {
1412 crate::OwnedValue::from_allocation(allocation, output_type.clone())
1413 };
1414 drop(arguments);
1415 drop(closure);
1416 unsafe { destination.push(stack, &mut result) };
1417 Ok(())
1418 }
1419 Err(error) => {
1420 drop(arguments);
1421 drop(closure);
1422 Err(error)
1423 }
1424 }
1425 });
1426 self.replace(arity + 1, result_type);
1427 Ok(())
1428 }
1429
1430 pub fn join2(&mut self, yes: Self, no: Self) -> Result<()> {
1444 let origin = self.stack_offset_before_top(1)?;
1445 self.stack_ids
1446 .last()
1447 .unwrap()
1448 .value_type
1449 .validate_opaque_leaf(TypeId::of::<bool>())?;
1450 ensure!(
1451 yes.base_stack_index == origin && no.base_stack_index == origin,
1452 "fragment origins must match"
1453 );
1454 ensure!(
1455 yes.argument_ids.is_empty() && no.argument_ids.is_empty(),
1456 "fragments cannot take preloaded arguments"
1457 );
1458 let result = yes.result_type()?.clone();
1459 ensure!(
1460 result.same_shape(no.result_type()?),
1461 "fragment result types must match"
1462 );
1463 let condition = ResultPlan::new(origin, ValueType::leaf::<bool>())?;
1464 self.segment.update_base_alignment(
1465 yes.segment
1466 .base_alignment()
1467 .max(no.segment.base_alignment()),
1468 );
1469 self.argument_reads.extend(yes.argument_reads);
1470 self.argument_reads.extend(no.argument_reads);
1471 self.segment.raw0_(move |stack| {
1472 let value = unsafe { condition.take_typed::<bool>(stack) };
1475 let branch = if value { &yes.segment } else { &no.segment };
1476 unsafe { branch.call0_stack(stack) }
1477 });
1478 self.replace(1, result);
1479 Ok(())
1480 }
1481}
1482
1483#[cfg(test)]
1484mod performance_tests {
1485 use super::*;
1486 use crate::{Array, ArrayOf, Cons, IntoCStackList, Nil};
1487 use std::cell::RefCell;
1488 use std::panic::{AssertUnwindSafe, catch_unwind};
1489 use std::rc::Rc;
1490
1491 #[repr(align(128))]
1493 struct CloneDrop {
1494 name: &'static str,
1495 events: Rc<RefCell<Vec<&'static str>>>,
1496 fail: Rc<Cell<bool>>,
1497 owned: bool,
1498 }
1499
1500 impl Clone for CloneDrop {
1501 fn clone(&self) -> Self {
1503 assert!(!self.fail.get(), "deliberate clone panic");
1504 Self {
1505 name: self.name,
1506 events: self.events.clone(),
1507 fail: self.fail.clone(),
1508 owned: true,
1509 }
1510 }
1511 }
1512
1513 impl Drop for CloneDrop {
1514 fn drop(&mut self) {
1516 if self.owned {
1517 self.events.borrow_mut().push(self.name);
1518 }
1519 }
1520 }
1521
1522 #[test]
1524 fn direct_clone_panic_restores_branch_and_ambient_ownership() {
1525 type Pair = Cons<ArrayOf<Leaf<CloneDrop>>, Cons<Leaf<CloneDrop>, Nil>>;
1526 for external_constant in [false, true] {
1527 let events = Rc::new(RefCell::new(Vec::new()));
1528 let fail = Rc::new(Cell::new(true));
1529 let leaf = |name, fail: Rc<Cell<bool>>, owned| CloneDrop {
1530 name,
1531 events: events.clone(),
1532 fail,
1533 owned,
1534 };
1535 let descriptor = ValueType::tuple(vec![
1536 AssociatedType {
1537 offset: 0,
1538 value_type: ValueType::array(ValueType::cloneable_leaf::<CloneDrop>()),
1539 },
1540 AssociatedType {
1541 offset: 0,
1542 value_type: ValueType::cloneable_leaf::<CloneDrop>(),
1543 },
1544 ])
1545 .unwrap();
1546 let mut source = Some(
1547 OwnedValue::new::<Pair>(
1548 (
1549 Array::from_vec(vec![leaf("child", Rc::new(Cell::new(false)), false)]),
1550 leaf("panic", fail.clone(), false),
1551 )
1552 .into_c_stack_list(),
1553 )
1554 .with_value_type(descriptor.clone())
1555 .unwrap(),
1556 );
1557 let mut body = DynSegment::new::<()>();
1558 for name in ["outer-a", "outer-b"] {
1559 let log = events.clone();
1560 body.op0(move || CloneDrop {
1561 name,
1562 events: log.clone(),
1563 fail: Rc::new(Cell::new(false)),
1564 owned: true,
1565 });
1566 }
1567 body.just(true);
1568 let mut yes = body.new_fragment();
1569 let log = events.clone();
1570 yes.op0(move || CloneDrop {
1571 name: "branch",
1572 events: log.clone(),
1573 fail: Rc::new(Cell::new(false)),
1574 owned: true,
1575 });
1576 if external_constant {
1577 yes.just_external(descriptor, Box::new(source.take().unwrap()))
1578 .unwrap();
1579 } else {
1580 yes.push_argument(0, descriptor).unwrap();
1581 }
1582 yes.drop_value().unwrap();
1583 yes.drop_value().unwrap();
1584 yes.just(9u32);
1585 let mut no = body.new_fragment();
1586 no.just(3u32);
1587 body.join2(yes, no).unwrap();
1588 body.op3(|_: CloneDrop, _: CloneDrop, result: u32| result)
1589 .unwrap();
1590
1591 let pointers: Vec<_> = source.iter().map(OwnedValue::as_ptr).collect();
1592 for _ in 0..2 {
1593 let mut stack = RawStack::with_base_alignment(body.segment.base_alignment());
1594 let _arguments = ArgumentGuard::install(&pointers);
1595 assert!(
1597 catch_unwind(AssertUnwindSafe(|| unsafe {
1598 body.segment.call0_stack(&mut stack)
1599 }))
1600 .is_err()
1601 );
1602 assert_eq!(stack.len(), 0);
1603 assert_eq!(
1604 &*events.borrow(),
1605 &["child", "branch", "outer-b", "outer-a"]
1606 );
1607 events.borrow_mut().clear();
1608 }
1609 fail.set(false);
1610 let inputs: Vec<&dyn Any> = source.iter().map(|owner| owner as &dyn Any).collect();
1611 assert_eq!(body.call_dyn::<u32>(&inputs).unwrap(), 9);
1612 assert_eq!(
1613 events
1614 .borrow()
1615 .iter()
1616 .filter(|name| **name == "child")
1617 .count(),
1618 1
1619 );
1620 assert_eq!(
1621 events
1622 .borrow()
1623 .iter()
1624 .filter(|name| **name == "panic")
1625 .count(),
1626 1
1627 );
1628 }
1629 }
1630
1631 #[test]
1633 fn inconsistent_reads_precede_body_effects() {
1634 let effects = Rc::new(Cell::new(0));
1635 let counter = effects.clone();
1636 let mut body = DynSegment::new::<()>();
1637 body.op0(move || counter.set(counter.get() + 1));
1638 body.drop_value().unwrap();
1639 body.push_arg::<i32>(0);
1640 body.drop_value().unwrap();
1641 body.push_arg::<String>(0);
1642 body.drop_value().unwrap();
1643 body.just(7u32);
1644 for _ in 0..2 {
1645 let error = body.call_dyn::<u32>(&[&1i32]).unwrap_err();
1646 assert!(error.to_string().contains("inconsistent argument 0"));
1647 assert_eq!(effects.get(), 0);
1648 }
1649 }
1650
1651 #[test]
1653 fn captureless_operations_retain_no_instruction_state() {
1654 let mut body = DynSegment::new::<()>();
1655 body.just(1i32);
1656 let initial = body.segment.storage_size();
1657 for _ in 0..4_000 {
1658 body.op1(|value: i32| value).unwrap();
1659 }
1660 assert_eq!(body.segment.storage_size(), initial);
1661 assert_eq!(body.call0::<i32>().unwrap(), 1);
1662 }
1663
1664 #[test]
1666 fn large_literal_arrays_preserve_values() {
1667 for length in [1_000, 2_000, 4_000] {
1668 let mut body = DynSegment::new::<()>();
1669 for _ in 0..length {
1670 body.just(1i32);
1671 }
1672 body.make_array(length, 0).unwrap();
1673 let output = body.call_dyn_value(&[]).unwrap();
1674 assert_eq!(
1675 output
1676 .try_into_typed::<crate::ArrayOf<Leaf<i32>>>()
1677 .unwrap()
1678 .len(),
1679 length
1680 );
1681 }
1682 }
1683
1684 #[test]
1686 fn large_cloned_arrays_preserve_values() {
1687 for length in [1_000, 2_000, 4_000] {
1688 for external_constant in [false, true] {
1689 let mut body = DynSegment::new::<()>();
1690 for _ in 0..length {
1691 if external_constant {
1692 body.just_external(ValueType::cloneable_leaf::<i32>(), Box::new(1i32))
1693 .unwrap();
1694 } else {
1695 body.push_arg::<i32>(0);
1696 }
1697 }
1698 body.make_array(length, 0).unwrap();
1699 let result = body.call_dyn_value(&[&1i32]).unwrap();
1700 assert_eq!(
1701 result.try_into_typed::<ArrayOf<Leaf<i32>>>().unwrap().len(),
1702 length
1703 );
1704 }
1705 }
1706 }
1707
1708 #[test]
1710 fn prepared_callees_avoid_temporary_operand_allocations() {
1711 use crate::{Callable, FnOf};
1712
1713 type Args = Cons<Leaf<u64>, Nil>;
1714 type Function = FnOf<Args, Leaf<u64>>;
1715 fn increment(args: &<Args as ValueRepr>::Value) -> Result<u64> {
1717 args.1.checked_add(1).ok_or_else(|| anyhow!("overflow"))
1718 }
1719 let mut interpreted = DynSegment::new::<()>();
1720 interpreted.push_arg::<u64>(0);
1721 interpreted.just(1u64);
1722 interpreted
1723 .op2r(|a: u64, b: u64| a.checked_add(b).ok_or_else(|| anyhow!("overflow")))
1724 .unwrap();
1725 let interpreted =
1726 Callable::from_prepared(interpreted.prepare::<Args, Leaf<u64>>().unwrap());
1727 for function in [Callable::from_fn(increment), interpreted] {
1728 for calls in [1u64, 10, 100] {
1729 let mut body = DynSegment::new::<()>();
1730 body.just(17u64);
1731 for _ in 0..calls {
1732 let function = function.clone();
1733 body.op0_repr::<Function, _>(move || function.clone())
1734 .unwrap();
1735 body.push_arg::<u64>(0);
1736 body.invoke_closure(1).unwrap();
1737 body.op2r(|a: u64, b: u64| a.checked_add(b).ok_or_else(|| anyhow!("overflow")))
1738 .unwrap();
1739 }
1740 let mut prepared = body.prepare::<Args, Leaf<u64>>().unwrap();
1741 let args = (7u64,).into_c_stack_list();
1742 let expected = 17 + calls * 8;
1743 assert_eq!(prepared.call(&args).unwrap(), expected);
1744 crate::owned_value::ALLOCATIONS.with(|count| count.set(0));
1745 for _ in 0..100 {
1746 assert_eq!(prepared.call(&args).unwrap(), expected);
1747 }
1748 let allocations = crate::owned_value::ALLOCATIONS.with(Cell::get);
1749 assert_eq!(allocations, 200 * calls as usize, "{calls} calls");
1750 }
1751 }
1752 }
1753
1754 #[test]
1756 fn prepared_indices_avoid_temporary_operand_allocations() {
1757 for accesses in [1u64, 10, 100] {
1758 let mut body = DynSegment::new::<()>();
1759 body.just(17u64);
1760 for _ in 0..accesses {
1761 let array = Array::from_vec(vec![7u64]);
1762 body.op0_repr::<ArrayOf<Leaf<u64>>, _>(move || array.clone())
1763 .unwrap();
1764 body.push_arg::<i32>(0);
1765 body.array_index::<i32>().unwrap();
1766 body.op2r(|a: u64, b: u64| a.checked_add(b).ok_or_else(|| anyhow!("overflow")))
1767 .unwrap();
1768 }
1769 let mut prepared = body.prepare::<Cons<Leaf<i32>, Nil>, Leaf<u64>>().unwrap();
1770 let args = (0i32,).into_c_stack_list();
1771 let expected = 17 + accesses * 7;
1772 assert_eq!(prepared.call(&args).unwrap(), expected);
1773 crate::owned_value::ALLOCATIONS.with(|count| count.set(0));
1774 for _ in 0..100 {
1775 assert_eq!(prepared.call(&args).unwrap(), expected);
1776 }
1777 let allocations = crate::owned_value::ALLOCATIONS.with(Cell::get);
1778 assert_eq!(allocations, 200 * accesses as usize, "{accesses} accesses");
1779 }
1780 }
1781
1782 #[test]
1784 fn prepared_callee_moves_preserve_error_and_unwind_cleanup() {
1785 use crate::{Callable, FnOf};
1786
1787 type Args = Cons<Leaf<CloneDrop>, Nil>;
1788 type Function = FnOf<Args, Leaf<u64>>;
1789 let events = Rc::new(RefCell::new(Vec::new()));
1790 let mode = Rc::new(Cell::new(0u8));
1791 let leaf = |name| CloneDrop {
1792 name,
1793 events: events.clone(),
1794 fail: Rc::new(Cell::new(false)),
1795 owned: true,
1796 };
1797 let mut body = DynSegment::new::<()>();
1798 let ambient = leaf("ambient");
1799 body.op0(move || ambient.clone());
1800 body.just(9u8);
1801 let callee_events = events.clone();
1802 let callee_mode = mode.clone();
1803 body.op0_repr::<Function, _>(move || {
1804 let sentinel = CloneDrop {
1805 name: "callee",
1806 events: callee_events.clone(),
1807 fail: Rc::new(Cell::new(false)),
1808 owned: true,
1809 };
1810 let mode = callee_mode.clone();
1811 let mut inner = DynSegment::new::<()>();
1812 inner.push_arg::<CloneDrop>(0);
1813 inner
1814 .op1r(move |argument: CloneDrop| {
1815 let _sentinel = &sentinel;
1816 drop(argument);
1817 match mode.get() {
1818 0 => Ok(7u64),
1819 1 => Err(anyhow!("callee failure")),
1820 _ => panic!("callee panic"),
1821 }
1822 })
1823 .unwrap();
1824 Callable::from_prepared(inner.prepare::<Args, Leaf<u64>>().unwrap())
1825 })
1826 .unwrap();
1827 let argument = leaf("argument");
1828 body.op0(move || argument.clone());
1829 body.invoke_closure(1).unwrap();
1830 body.op3(|ambient: CloneDrop, marker: u8, value: u64| {
1831 assert_eq!(marker, 9);
1832 drop(ambient);
1833 value
1834 })
1835 .unwrap();
1836 let mut prepared = body.prepare::<Nil, Leaf<u64>>().unwrap();
1837 let args = ().into_c_stack_list();
1838 for _ in 0..2 {
1839 for next in [0, 1, 0, 2, 0] {
1840 mode.set(next);
1841 let result = catch_unwind(AssertUnwindSafe(|| prepared.call(&args)));
1842 match next {
1843 0 => assert_eq!(result.unwrap().unwrap(), 7),
1844 1 => assert_eq!(result.unwrap().unwrap_err().to_string(), "callee failure"),
1845 _ => assert!(result.is_err()),
1846 }
1847 let expected = if next == 2 {
1848 ["argument", "callee", "argument", "ambient"]
1849 } else {
1850 ["argument", "argument", "callee", "ambient"]
1851 };
1852 assert_eq!(&*events.borrow(), &expected);
1853 events.borrow_mut().clear();
1854 }
1855 }
1856 }
1857
1858 #[repr(align(128))]
1860 struct TrackedIndex {
1861 mode: u8,
1862 events: Rc<RefCell<Vec<&'static str>>>,
1863 }
1864
1865 impl TryFrom<TrackedIndex> for usize {
1866 type Error = ();
1867
1868 fn try_from(index: TrackedIndex) -> std::result::Result<Self, Self::Error> {
1870 match index.mode {
1871 0 => Ok(1),
1872 1 => Err(()),
1873 2 => panic!("index conversion panic"),
1874 _ => Ok(2),
1875 }
1876 }
1877 }
1878
1879 impl Drop for TrackedIndex {
1880 fn drop(&mut self) {
1882 self.events.borrow_mut().push("index");
1883 }
1884 }
1885
1886 #[test]
1888 fn prepared_index_moves_preserve_error_and_unwind_cleanup() {
1889 let events = Rc::new(RefCell::new(Vec::new()));
1890 let mode = Rc::new(Cell::new(0u8));
1891 let mut body = DynSegment::new::<()>();
1892 let log = events.clone();
1893 body.op0(move || CloneDrop {
1894 name: "ambient",
1895 events: log.clone(),
1896 fail: Rc::new(Cell::new(false)),
1897 owned: true,
1898 });
1899 body.just(9u8);
1900 let origin = body.current_stack_offset();
1901 for name in ["first", "last"] {
1902 let log = events.clone();
1903 body.op0(move || CloneDrop {
1904 name,
1905 events: log.clone(),
1906 fail: Rc::new(Cell::new(false)),
1907 owned: true,
1908 });
1909 }
1910 body.make_array(2, origin).unwrap();
1911 let index_events = events.clone();
1912 let index_mode = mode.clone();
1913 body.op0(move || TrackedIndex {
1914 mode: index_mode.get(),
1915 events: index_events.clone(),
1916 });
1917 body.array_index::<TrackedIndex>().unwrap();
1918 body.op3(|ambient: CloneDrop, marker: u8, value: CloneDrop| {
1919 assert_eq!(marker, 9);
1920 drop(ambient);
1921 value
1922 })
1923 .unwrap();
1924 let mut prepared = body.prepare::<Nil, Leaf<CloneDrop>>().unwrap();
1925 let args = ().into_c_stack_list();
1926 for _ in 0..2 {
1927 for next in [0, 1, 0, 2, 0, 3, 0] {
1928 mode.set(next);
1929 let result = catch_unwind(AssertUnwindSafe(|| prepared.call(&args)));
1930 match next {
1931 0 => drop(result.unwrap().unwrap()),
1932 1 => assert_eq!(
1933 result.unwrap().err().unwrap().to_string(),
1934 "array index is negative or unrepresentable"
1935 ),
1936 2 => assert!(result.is_err()),
1937 _ => assert!(
1938 result
1939 .unwrap()
1940 .err()
1941 .unwrap()
1942 .to_string()
1943 .contains("out of bounds")
1944 ),
1945 }
1946 let expected = if next == 0 {
1947 ["index", "first", "ambient", "last"]
1948 } else {
1949 ["index", "last", "first", "ambient"]
1950 };
1951 assert_eq!(&*events.borrow(), &expected);
1952 events.borrow_mut().clear();
1953 }
1954 }
1955 }
1956
1957 #[test]
1959 fn prepared_branches_avoid_temporary_allocations() {
1960 type Inputs = Cons<Leaf<bool>, Cons<Leaf<bool>, Nil>>;
1961 for kind in ["if", "&&", "||"] {
1962 for conditions in [1u64, 10, 100] {
1963 let mut body = DynSegment::new::<()>();
1964 body.just(17u64);
1965 for _ in 0..conditions {
1966 body.push_arg::<bool>(0);
1967 let mut yes = body.new_fragment();
1968 let mut no = body.new_fragment();
1969 match kind {
1970 "if" => {
1971 yes.push_arg::<bool>(1);
1972 no.push_arg::<bool>(1);
1973 no.op1(|value: bool| !value).unwrap();
1974 }
1975 "&&" => {
1976 yes.push_arg::<bool>(1);
1977 no.just(false);
1978 }
1979 "||" => {
1980 yes.just(true);
1981 no.push_arg::<bool>(1);
1982 }
1983 _ => unreachable!(),
1984 }
1985 body.join2(yes, no).unwrap();
1986 body.op2r(|total: u64, value: bool| {
1987 total
1988 .checked_add(u64::from(value))
1989 .ok_or_else(|| anyhow!("overflow"))
1990 })
1991 .unwrap();
1992 }
1993 let mut prepared = body.prepare::<Inputs, Leaf<u64>>().unwrap();
1994 for condition in [false, true] {
1995 for rhs in [false, true] {
1996 let value = match kind {
1997 "if" => {
1998 if condition {
1999 rhs
2000 } else {
2001 !rhs
2002 }
2003 }
2004 "&&" => condition && rhs,
2005 "||" => condition || rhs,
2006 _ => unreachable!(),
2007 };
2008 let expected = 17 + conditions * u64::from(value);
2009 let inputs = (condition, rhs).into_c_stack_list();
2010 assert_eq!(prepared.call(&inputs).unwrap(), expected);
2011 crate::owned_value::ALLOCATIONS.with(|count| count.set(0));
2012 for _ in 0..100 {
2013 assert_eq!(prepared.call(&inputs).unwrap(), expected);
2014 }
2015 let allocations = crate::owned_value::ALLOCATIONS.with(Cell::get);
2016 assert_eq!(allocations, 0, "{kind}, {conditions} conditions");
2017 }
2018 }
2019 }
2020 }
2021 }
2022
2023 #[test]
2025 fn scalar_cloning_avoids_temporary_allocations() {
2026 for reads in [1, 10, 100] {
2027 for external_constant in [false, true] {
2028 let mut body = DynSegment::new::<()>();
2029 for read in 0..reads {
2030 if external_constant {
2031 body.just_external(ValueType::cloneable_leaf::<i32>(), Box::new(1i32))
2032 .unwrap();
2033 } else {
2034 body.push_arg::<i32>(0);
2035 }
2036 if read != 0 {
2037 body.op2r(|a: i32, b: i32| {
2038 a.checked_add(b).ok_or_else(|| anyhow!("overflow"))
2039 })
2040 .unwrap();
2041 }
2042 }
2043 let mut prepared = body.prepare::<Cons<Leaf<i32>, Nil>, Leaf<i32>>().unwrap();
2044 let input = (1i32,).into_c_stack_list();
2045 assert_eq!(prepared.call(&input).unwrap(), reads);
2046 crate::owned_value::ALLOCATIONS.with(|count| count.set(0));
2047 for _ in 0..100 {
2048 assert_eq!(prepared.call(&input).unwrap(), reads);
2049 }
2050 let allocations = crate::owned_value::ALLOCATIONS.with(Cell::get);
2051 eprintln!(
2052 "{reads} reads, constant={external_constant}: {allocations} temporary allocations"
2053 );
2054 assert_eq!(allocations, 0);
2055 }
2056 }
2057 }
2058
2059 #[test]
2061 fn repeated_read_preflight_is_linear() {
2062 for reads in [1_000, 2_000, 4_000] {
2063 let mut body = DynSegment::new::<()>();
2064 body.push_arg::<i32>(0);
2065 for _ in 1..reads {
2066 body.push_arg::<i32>(0);
2067 body.op2r(|a: i32, b: i32| a.checked_add(b).ok_or_else(|| anyhow!("overflow")))
2068 .unwrap();
2069 }
2070 for _ in 0..2 {
2071 crate::value_type::SHAPE_COMPARISONS.with(|count| count.set(0));
2072 assert_eq!(body.call_dyn::<i32>(&[&1i32]).unwrap(), reads);
2073 let comparisons = crate::value_type::SHAPE_COMPARISONS.with(Cell::get);
2074 eprintln!("{reads} checked reads: {comparisons} descriptor comparisons");
2075 assert!(comparisons <= 4 * reads as usize + 16);
2076 assert!(body.call_dyn::<i32>(&[]).is_err());
2077 assert!(body.call_dyn::<i32>(&[&1u32]).is_err());
2078 }
2079 let mut prepared = body.prepare::<Cons<Leaf<i32>, Nil>, Leaf<i32>>().unwrap();
2080 crate::value_type::SHAPE_COMPARISONS.with(|count| count.set(0));
2081 assert_eq!(prepared.call(&(1i32,).into_c_stack_list()).unwrap(), reads);
2082 crate::value_type::SHAPE_COMPARISONS.with(|count| assert_eq!(count.get(), 0));
2083 }
2084 }
2085}