Make typed source index paths iterative and strengthen adversarial controls

crates/oa-core/src/language/analysis_source/index.rs

@@ -1,23 +1,83 @@
 //! Temporary typed source witnesses. No arena index becomes a model identity.
 //! The staged entry point parses each supplied typed file once. A future bundle
-//! parser can pass its already parsed units/tokens to the private builder.
+//! parser must retain the same opaque parse/lower provenance. Occurrence
+//! completeness trusts that lowerer; the private builder does not prove the
+//! presence of every syntax reference in a caller-fabricated Lowered value.
 use super::cursor::Cursor;
 use super::lower::{self, Child, Field, Lowered, Role, Target, Unit};
 use super::{Budget, Error, Span, declarations, error, lex};
 use crate::{Id, Project, analysis as a};
+use std::cmp::Ordering;
 use std::collections::{BTreeMap, BTreeSet};
+use std::fmt;
 use std::sync::Arc;
 
 /// Persistent structural path: cloning shares ancestors, and equality/order
 /// compare the complete edge sequence, never allocation or arena identities.
 /// Depth is checked against the inherited frame limit before allocation.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)]
+/// Comparison and Debug walk leaf-to-root without allocating or recursing;
+/// this private total order need not be root-first lexicographic order.
+#[derive(Clone, Default)]
 pub(super) struct Path(Option<Arc<Step>>);
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
 struct Step {
     parent: Path,
     edge: Child,
 }
+impl PartialEq for Path {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == Ordering::Equal
+    }
+}
+impl Eq for Path {}
+impl PartialOrd for Path {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl Ord for Path {
+    fn cmp(&self, other: &Self) -> Ordering {
+        let (mut left, mut right) = (self, other);
+        loop {
+            match (left.0.as_deref(), right.0.as_deref()) {
+                (Some(a), Some(b)) => {
+                    let order = a.edge.cmp(&b.edge);
+                    if order != Ordering::Equal {
+                        return order;
+                    }
+                    (left, right) = (&a.parent, &b.parent);
+                }
+                (None, None) => return Ordering::Equal,
+                (None, Some(_)) => return Ordering::Less,
+                (Some(_), None) => return Ordering::Greater,
+            }
+        }
+    }
+}
+impl fmt::Debug for Path {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Debug intentionally shows the same leaf-to-root order as comparison.
+        let mut list = formatter.debug_list();
+        let mut path = self;
+        while let Some(step) = &path.0 {
+            list.entry(&step.edge);
+            path = &step.parent;
+        }
+        list.finish()
+    }
+}
+impl Drop for Path {
+    fn drop(&mut self) {
+        let mut next = self.0.take();
+        while let Some(link) = next {
+            // Shared ancestors remain owned elsewhere. The final owner peels
+            // unique links here, taking the parent before Step is dropped.
+            let Some(mut step) = Arc::into_inner(link) else {
+                break;
+            };
+            next = step.parent.0.take();
+        }
+    }
+}
 impl Path {
     pub fn edges(&self) -> Vec<Child> {
         let mut result = Vec::new();
@@ -119,6 +179,9 @@ impl<'s> Index<'s> {
         Ok((index, lowered))
     }
 
+    // Same-parse/lower internal seam only. Checks below reject damaged supplied
+    // sites but occurrence completeness is an invariant of our own lowerer.
+    // Do not expose a constructor accepting independently fabricated Lowered.
     fn build(
         ordinary: &Project,
         sources: &'s BTreeMap<usize, String>,

crates/oa-core/src/language/analysis_source/index/tests.rs

@@ -206,9 +206,30 @@ fn arena_rejects_forward_cycles_and_cross_owner_field_file_parents() {
 #[test]
 fn arena_rejects_duplicate_sites_occurrences_and_wrong_callee_labels() {
     for damage in 0..4 {
-        let sources = sources();
+        let mut sources = sources();
+        // The other callee is fully valid and reuses both parameter identities
+        // and spellings. Catalog/token validation must not explain rejection.
+        sources.insert(
+            31,
+            r#"calculation other "Other" id "00000000-0000-0000-0000-00000000cafe" {
+ input a id "28ff40b0-1b8f-4735-8536-5a6ab21b56cb" : Quantity(@duration);
+ input b id "7a47cc1e-8ae5-4551-b2d8-8a68108e0976" : Quantity(@duration);
+ result Quantity(@duration); body $a + $b;
+}"#
+            .into(),
+        );
         let mut work = budget();
         let (units, witnesses, mut lowered) = raw(&sources, &mut work);
+        let valid = Index::build(
+            &ordinary(),
+            &sources,
+            &units,
+            &witnesses,
+            &lowered,
+            &mut work,
+        )
+        .unwrap();
+        let mut expected_mismatch_span = None;
         match damage {
             0 => {
                 let site = &lowered.expressions[0];
@@ -238,9 +259,21 @@ fn arena_rejects_duplicate_sites_occurrences_and_wrong_callee_labels() {
                     .iter_mut()
                     .find(|row| row.role == Role::ArgumentLabel)
                     .unwrap();
-                if let Target::Parameter { calculation, .. } = &mut label.target {
+                let original = label.target;
-                    *calculation = ordinary().id;
+                let Target::Parameter {
-                }
+                    calculation,
+                    parameter,
+                } = &mut label.target
+                else {
+                    panic!("argument label must target a parameter");
+                };
+                let other = Target::Parameter {
+                    calculation: Id::from_u128(0xcafe),
+                    parameter: *parameter,
+                };
+                assert_eq!(valid.names[&original], valid.names[&other]);
+                *calculation = Id::from_u128(0xcafe);
+                expected_mismatch_span = Some(label.span);
             }
             _ => {
                 let row = lowered
@@ -251,18 +284,22 @@ fn arena_rejects_duplicate_sites_occurrences_and_wrong_callee_labels() {
                 row.field = Field::Override(ordinary().id);
             }
         }
-        assert!(
+        let failure = Index::build(
-            Index::build(
+            &ordinary(),
-                &ordinary(),
+            &sources,
-                &sources,
+            &units,
-                &units,
+            &witnesses,
-                &witnesses,
+            &lowered,
-                &lowered,
+            &mut work,
-                &mut work
+        )
-            )
+        .unwrap_err();
-            .is_err(),
+        if let Some(span) = expected_mismatch_span {
-            "damage {damage}"
+            assert_eq!(
-        );
+                failure.message,
+                "Argument label crosses its called calculation."
+            );
+            assert_eq!(failure.span, span);
+        }
     }
 }
 
@@ -324,3 +361,270 @@ fn index_has_shared_budget_and_never_stabilizes_missing_ids() {
     assert!(Index::parse(&ordinary(), &sources(), &mut work).is_err());
     assert!(work.active().is_err());
 }
+
+fn on_small_stack(work: impl FnOnce() + Send + 'static) {
+    // Match the established core language_nesting_tests worker-stack request.
+    // This is a requested platform stack size, not a measured peak-byte claim.
+    std::thread::Builder::new()
+        .name("analysis-index-128k".into())
+        .stack_size(128 * 1024)
+        .spawn(work)
+        .unwrap()
+        .join()
+        .unwrap();
+}
+
+fn chain(first: Child, edges: usize) -> Path {
+    let mut path = Path::default();
+    for position in 0..edges {
+        path = Path(Some(Arc::new(Step {
+            parent: path,
+            edge: if position == 0 { first } else { Child::Operand },
+        })));
+    }
+    path
+}
+
+#[test]
+fn maximum_index_paths_compare_lookup_debug_and_drop_on_128k_stack() {
+    on_small_stack(|| {
+        let frames = super::super::budget::Limits::source8(ParseLimits::UNLIMITED).frames;
+        assert_eq!(frames, 256);
+        let edges = frames - 1; // root has frame depth one
+        let mut work = budget();
+        work.frame(frames, Span::default()).unwrap();
+        let left = chain(Child::Left, edges);
+        let equal = chain(Child::Left, edges); // independent allocations
+        let right = chain(Child::Right, edges);
+        assert!(!Arc::ptr_eq(
+            left.0.as_ref().unwrap(),
+            equal.0.as_ref().unwrap()
+        ));
+        assert_eq!(left, equal);
+        assert_eq!(left.partial_cmp(&equal), Some(Ordering::Equal));
+        // The difference is at the root-most edge: leaf-first walks every link.
+        assert_eq!(left.cmp(&right), Child::Left.cmp(&Child::Right));
+        assert_eq!(right.cmp(&left), left.cmp(&right).reverse());
+        assert!(Path::default() < left);
+        assert!(chain(Child::Operand, edges - 1) < chain(Child::Operand, edges));
+        let shared = left.clone();
+        assert!(Arc::ptr_eq(
+            left.0.as_ref().unwrap(),
+            shared.0.as_ref().unwrap()
+        ));
+        let leaf = Arc::downgrade(left.0.as_ref().unwrap());
+        let key = |path| ExpressionKey {
+            owner: Id::from_u128(1),
+            field: Field::Body,
+            path,
+        };
+        let mut expressions = BTreeMap::new();
+        assert_eq!(expressions.insert(key(left.clone()), 1), None);
+        assert_eq!(expressions.insert(key(equal.clone()), 2), Some(1));
+        assert_eq!(expressions.insert(key(right.clone()), 3), None);
+        assert_eq!(expressions.get(&key(equal.clone())), Some(&2));
+        assert_eq!(expressions.get(&key(chain(Child::Right, edges))), Some(&3));
+        assert_eq!(expressions.len(), 2);
+        let occurrence = |path| OccurrenceKey {
+            owner: Id::from_u128(1),
+            field: Some(Field::Body),
+            path: Some(path),
+            role: TokenRole::Use(Role::ParameterUse),
+            target: Target::Parameter {
+                calculation: Id::from_u128(1),
+                parameter: a::ParameterId(Id::from_u128(2)),
+            },
+        };
+        let mut occurrences = BTreeSet::new();
+        assert!(occurrences.insert(occurrence(left.clone())));
+        assert!(!occurrences.insert(occurrence(equal.clone())));
+        assert!(occurrences.contains(&occurrence(chain(Child::Left, edges))));
+        let debug = format!("{left:?}");
+        assert_eq!(debug.matches("Operand").count(), edges - 1);
+        assert!(debug.ends_with("Left]"));
+        assert!(!format!("{expressions:?} {occurrences:?}").is_empty());
+        drop(expressions);
+        drop(occurrences);
+        drop(shared);
+        drop(left);
+        assert!(leaf.upgrade().is_none(), "maps/clones release their chain");
+        drop(equal);
+        drop(right);
+        // Dropping a child stops at a shared ancestor, which remains usable.
+        let parent = chain(Child::Left, edges - 1);
+        let ancestor = Arc::downgrade(parent.0.as_ref().unwrap());
+        let child = Path(Some(Arc::new(Step {
+            parent: parent.clone(),
+            edge: Child::Right,
+        })));
+        drop(child);
+        assert_eq!(parent.edges().len(), edges - 1);
+        drop(parent);
+        assert!(ancestor.upgrade().is_none());
+        // No other strong owners: final destruction must peel the full chain.
+        let unique = chain(Child::Left, edges);
+        let final_leaf = Arc::downgrade(unique.0.as_ref().unwrap());
+        drop(unique);
+        assert!(final_leaf.upgrade().is_none());
+    });
+}
+
+#[test]
+fn deep_parse_index_plan_and_failed_build_cleanup_on_128k_stack() {
+    on_small_stack(|| {
+        use super::super::update::Plan;
+        let deep = format!("{}$a{}", "abs(".repeat(127), ")".repeat(127));
+        let source = format!(
+            r#"calculation f "Deep 雪" id "00000000-0000-0000-0000-000000000081" {{
+ input a id "00000000-0000-0000-0000-000000000083" : Number;
+ result Number; body {deep};
+}}
+calculation caller "Caller" id "00000000-0000-0000-0000-000000000082" {{
+ result Number; body @f(a: 1);
+}}
+"#
+        );
+        let sources = BTreeMap::from([(47, source.clone())]);
+        let skeleton = ordinary();
+        let mut work = budget();
+        let (index, lowered) = Index::parse(&skeleton, &sources, &mut work).unwrap();
+        assert_eq!(work.usage.bytes, source.len());
+        assert_eq!(index.expressions.len(), 130); // 128 unary/leaf + call/argument
+        assert_eq!(
+            index.expressions.keys().map(|k| k.path.edges().len()).max(),
+            Some(127)
+        );
+        let renames = BTreeMap::from([
+            (Target::Global(Id::from_u128(0x81)), "renamed_deep".into()),
+            (
+                Target::Parameter {
+                    calculation: Id::from_u128(0x81),
+                    parameter: a::ParameterId(Id::from_u128(0x83)),
+                },
+                "renamed_arg".into(),
+            ),
+        ]);
+        let usage = work.usage;
+        let plan = Plan::renames(&index, &renames, &work).unwrap();
+        assert_eq!(plan.edit_count(), 5); // declaration/call + input/use/label
+        let output = plan.apply(&sources, &work).unwrap();
+        assert_eq!(work.usage, usage);
+        // Independent complete original-token oracle; no index-derived offsets.
+        let expected_source = source
+            .replace("calculation f ", "calculation renamed_deep ")
+            .replace("input a id", "input renamed_arg id")
+            .replace("$a", "$renamed_arg")
+            .replace("@f(a: 1)", "@renamed_deep(renamed_arg: 1)");
+        assert_eq!(output, BTreeMap::from([(47, expected_source)]));
+        let (renamed_index, renamed) = Index::parse(&skeleton, &output, &mut budget()).unwrap();
+        assert_eq!(
+            index.occurrences.keys().collect::<Vec<_>>(),
+            renamed_index.occurrences.keys().collect::<Vec<_>>()
+        );
+        // Compare complete independently constructed models, including deepest
+        // parameter identity and the call's parameter-keyed argument map.
+        for (lowered, renamed) in [(lowered, false), (renamed, true)] {
+            let mut expected = skeleton.clone();
+            let parameter = a::ParameterId(Id::from_u128(0x83));
+            let mut expression = a::Expr::Parameter(parameter);
+            for _ in 0..127 {
+                expression = a::Expr::Unary {
+                    op: a::UnaryOp::Absolute,
+                    operand: Box::new(expression),
+                };
+            }
+            for (id, alias, name, parameters, expression) in [
+                (
+                    0x81,
+                    if renamed { "renamed_deep" } else { "f" },
+                    "Deep 雪",
+                    vec![a::Parameter {
+                        id: parameter,
+                        name: if renamed { "renamed_arg" } else { "a" }.into(),
+                        value_type: a::AnalysisType::Scalar(a::ScalarType::Number),
+                    }],
+                    expression,
+                ),
+                (
+                    0x82,
+                    "caller",
+                    "Caller",
+                    Vec::new(),
+                    a::Expr::Call {
+                        definition: a::DefinitionRef::Local(Id::from_u128(0x81)),
+                        arguments: BTreeMap::from([(
+                            parameter,
+                            a::Expr::Number(crate::analysis_numeric::Exact::from_integer(1)),
+                        )]),
+                    },
+                ),
+            ] {
+                let id = Id::from_u128(id);
+                expected.definitions.insert(
+                    id,
+                    a::Definition {
+                        id,
+                        alias: alias.into(),
+                        name: name.into(),
+                        description: String::new(),
+                        body: a::DefinitionBody::Calculation(a::CalculationDefinition {
+                            parameters,
+                            result: a::AnalysisType::Scalar(a::ScalarType::Number),
+                            body: a::Binding::Expression(expression),
+                        }),
+                    },
+                );
+            }
+            let mut actual = skeleton.clone();
+            actual.definitions.extend(lowered.definitions);
+            actual.elements.extend(lowered.elements);
+            assert_eq!(actual, expected);
+            a::admit(&actual, a::AdmissionLimits::REFERENCE).unwrap();
+            drop(actual);
+            drop(expected);
+        }
+        drop(plan);
+        drop(renamed_index);
+        drop(index);
+        // Failure after the deepest keys have been allocated must clean up on
+        // the same stack; no modified Lowered escapes this private test seam.
+        let mut work = budget();
+        let (units, witnesses, mut lowered) = raw(&sources, &mut work);
+        let deepest = lowered
+            .expressions
+            .iter()
+            .find(|site| &sources[&site.file][site.span.start..site.span.end] == "$a")
+            .unwrap();
+        lowered.expressions.push(lower::ExpressionSite {
+            file: deepest.file,
+            owner: deepest.owner,
+            field: deepest.field,
+            parent: deepest.parent,
+            span: deepest.span,
+        });
+        let failure =
+            Index::build(&skeleton, &sources, &units, &witnesses, &lowered, &mut work).unwrap_err();
+        assert_eq!(failure.message, "Duplicate qualified expression path.");
+        lowered.expressions.pop();
+        let mut limits = super::super::budget::Limits::source8(ParseLimits::UNLIMITED);
+        limits.frames = 127;
+        let mut bounded = Budget::new(limits).unwrap();
+        let failure = Index::build(
+            &skeleton,
+            &sources,
+            &units,
+            &witnesses,
+            &lowered,
+            &mut bounded,
+        )
+        .unwrap_err();
+        assert!(failure.message.contains("syntax frames"));
+        assert!(bounded.active().is_err());
+        drop(lowered);
+        drop(witnesses);
+        drop(units);
+        let rejected = BTreeMap::from([(47, source.replace(&deep, &format!("abs({deep})")))]);
+        let failure = Index::parse(&skeleton, &rejected, &mut budget()).unwrap_err();
+        assert!(failure.error.message.contains("logical depth"));
+    });
+}