diff --git a/crates/oa-core/src/language.rs b/crates/oa-core/src/language.rs index 2a5b6525..b9bb4d30 100644 --- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -1,8 +1,8 @@ //! Typed OA/SA language, tolerant source analysis, and explicit V1/V2/V3/V4 migration. +mod raw_tokens; #[path = "language/semantic_source.rs"] mod semantic_source; mod source3_bridge; -mod raw_tokens; // Staged syntax foundation only. Public source8 admission waits for complete // declaration lowering, source updates and the root-owned resource gate. #[cfg(test)] @@ -480,6 +480,8 @@ fn parse_document(source: &str, limits: ParseLimits) -> Result Result, Box> { let tokens = strict_tokens(source).map_err(|e| Box::new(diagnostic(source, e)))?; - let mut ends: Vec<_> = tokens.iter().map(|token| token.span.end).collect(); + edit_positions_from_tokens(source, &unit, |at| tokens[at], tokens.len(), None) +} + +fn edit_positions_from_tokens( + source: &str, + unit: &SourceUnit, + token: impl Fn(usize) -> Lexeme, + count: usize, + value_ends: Option<&BTreeMap>, +) -> Result, Box> { + let mut ends: Vec<_> = (0..count).map(|at| token(at).span.end).collect(); let mut stack = Vec::new(); - for (index, token) in tokens.iter().enumerate() { + for index in 0..count { + let token = token(index); match token.text(source) { "{" | "[" => stack.push(index), "}" | "]" => ends[stack.pop().expect("parser checked delimiters")] = token.span.end, _ => {} } } - struct Positions<'a> { - tokens: &'a [Lexeme], + struct Positions<'a, F> { + token_at: F, ends: Vec, + value_ends: Option<&'a BTreeMap>, } - impl Positions<'_> { + impl Lexeme> Positions<'_, F> { fn token(&self, start: usize) -> usize { - self.tokens - .partition_point(|token| token.span.start < start) + let mut low = 0; + let mut high = self.ends.len(); + while low < high { + let middle = low + (high - low) / 2; + if (self.token_at)(middle).span.start < start { + low = middle + 1; + } else { + high = middle; + } + } + low } fn fields(&self, fields: &BTreeMap) -> BTreeMap { fields .iter() .map(|(key, ast)| { let value = self.value(ast); - let start = self.tokens[self.token(value.span.start) - 1].span.start; - let end = self.tokens[self.token(value.span.end)].span.end; + let start = (self.token_at)(self.token(value.span.start) - 1).span.start; + let end = (self.token_at)(self.token(value.span.end)).span.end; ( key.clone(), EditField { @@ -1044,10 +1069,15 @@ fn edit_positions( let index = self.token(ast.offset()); let span = Span { start: ast.offset(), - end: match ast { - Ast::Reference(..) | Ast::HistoricalUuid(..) => self.tokens[index + 1].span.end, - _ => self.ends[index], - }, + end: self + .value_ends + .and_then(|ends| ends.get(&ast.offset()).copied()) + .unwrap_or_else(|| match ast { + Ast::Reference(..) | Ast::HistoricalUuid(..) => { + (self.token_at)(index + 1).span.end + } + _ => self.ends[index], + }), }; let kind = match ast { Ast::Object(fields, _) => EditKind::Record(self.fields(fields)), @@ -1062,11 +1092,12 @@ fn edit_positions( } } let positions = Positions { - tokens: &tokens, + token_at: &token, ends, + value_ends, }; unit.declarations - .into_iter() + .iter() .map(|declaration| { if declaration.id_insertion.is_some() { return Err(Box::new(diagnostic( @@ -1086,8 +1117,8 @@ fn edit_positions( let start = positions.token(declaration.span.start); let header = roles .iter() - .zip(&tokens[start..]) - .map(|(role, token)| (*role, token.span)) + .zip(start..count) + .map(|(role, at)| (*role, token(at).span)) .collect(); Ok(( declaration.id, @@ -1114,10 +1145,35 @@ struct Parser<'a> { cursor: usize, value_depth: usize, limits: ParseLimits, + #[cfg(test)] + mixed: Option>, } impl Parser<'_> { + fn token(&self, index: usize) -> Option { + #[cfg(test)] + if let Some(mixed) = &self.mixed { + return mixed.token(index); + } + self.tokens.get(index).copied() + } fn current(&self) -> Lexeme { - self.tokens[self.cursor.min(self.tokens.len() - 1)] + self.token(self.cursor).unwrap_or(Lexeme { + class: TokenClass::Punctuation, + span: Span { + start: self.source.len(), + end: self.source.len(), + }, + }) + } + fn syntax_entry(&mut self) -> Result<(), Error> { + #[cfg(test)] + { + let span = self.current().span; + if let Some(mixed) = &mut self.mixed { + mixed.entry(span)?; + } + } + Ok(()) } fn text(&self) -> &str { self.current().text(self.source) @@ -1154,6 +1210,10 @@ impl Parser<'_> { } fn optional_id(&mut self) -> Result { if self.text() != "id" { + #[cfg(test)] + if self.mixed.is_some() { + return Err(self.error("Private mixed syntax requires explicit IDs.")); + } return Ok(Id::new_v4()); } self.cursor += 1; @@ -1194,6 +1254,7 @@ impl Parser<'_> { Ok(fields) } fn field(&mut self) -> Result<(String, Ast, Span), Error> { + self.syntax_entry()?; let key = self.text().to_owned(); if !matches!( self.current().class, @@ -1204,15 +1265,19 @@ impl Parser<'_> { self.cursor += 1; let start = self.current().span.start; let value = self.value()?; - let end = self.tokens[self.cursor - 1].span.end; + let end = self + .token(self.cursor - 1) + .expect("consumed token") + .span + .end; self.take(";")?; Ok((key, value, Span { start, end })) } fn nested_header_kind(&self) -> Option { let kind = declaration_kind(self.text())?; - let alias = self.tokens.get(self.cursor + 1)?; - let following = self.tokens.get(self.cursor + 2)?; + let alias = self.token(self.cursor + 1)?; + let following = self.token(self.cursor + 2)?; // A node's ordinary `diagram @view;` field is not a declaration. if !crate::valid_alias(alias.text(self.source)) { return None; @@ -1227,14 +1292,37 @@ impl Parser<'_> { if self.value_depth >= 64 { return Err(self.error("Typed values may not nest more than 64 levels.")); } + self.syntax_entry()?; self.value_depth += 1; let result = self.value_inner(); self.value_depth -= 1; + #[cfg(test)] + if let Ok(ast) = &result { + let end = self + .token(self.cursor - 1) + .expect("consumed value") + .span + .end; + if let Some(mixed) = &mut self.mixed { + mixed.value_ends.insert(ast.offset(), end); + } + } result } fn value_inner(&mut self) -> Result { let token = self.current(); let offset = token.span.start; + #[cfg(test)] + if self.mixed.is_some() + && (token.class == TokenClass::Number || matches!(self.text(), "-" | "+")) + { + let value = self + .mixed + .as_ref() + .expect("mixed view") + .ordinary_number(&mut self.cursor)?; + return Ok(Ast::Literal(value, offset)); + } match self.text() { "uuid" => { self.cursor += 1; @@ -1332,16 +1420,31 @@ impl Parser<'_> { fn unit(&mut self, entry: bool, includes_allowed: bool) -> Result { let mut header = if entry { self.take("arcadia")?; - if !matches!(self.text(), "3" | "4" | "5" | "6" | "7") { - return Err( - self.error("Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.") - ); + #[cfg(test)] + let mixed_mode = self.mixed.is_some(); + #[cfg(not(test))] + let mixed_mode = false; + if if mixed_mode { + self.text() != "8" + } else { + !matches!(self.text(), "3" | "4" | "5" | "6" | "7") + } { + return Err(self.error(if mixed_mode { + "Private mixed syntax requires arcadia 8." + } else { + "Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7." + })); } let schema_version = self.text().parse().expect("checked source version"); self.cursor += 1; self.take(";")?; let start = self.current().span.start; self.take("project")?; + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.record(false, Span { start, end: start })?; + } + self.syntax_entry()?; let alias = self.identifier()?; let name = self.quoted()?; let id_insertion = (self.text() != "id").then_some(self.current().span.start); @@ -1382,6 +1485,17 @@ impl Parser<'_> { let mut open: Vec = Vec::new(); loop { let offset = self.current().span.start; + #[cfg(test)] + if let Some(mixed) = &self.mixed { + if mixed.typed_header(self.cursor, open.is_empty()) { + let owner = open.last().map(|&at| declarations[at].id); + self.mixed + .as_mut() + .expect("mixed view") + .parse_typed(&mut self.cursor, owner)?; + continue; + } + } if let Some(&owner) = open.last() { if self.text() == "}" { declarations[owner].body.end = offset; @@ -1425,6 +1539,14 @@ impl Parser<'_> { if !includes_allowed { return Err(self.error("An include requires a source bundle; single-string parsing never reads files.")); } + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.include(Span { + start: offset, + end: offset, + })?; + } + self.syntax_entry()?; self.cursor += 1; let path = self.quoted()?; self.take(";")?; @@ -1432,7 +1554,11 @@ impl Parser<'_> { path, Span { start: offset, - end: self.tokens[self.cursor - 1].span.end, + end: self + .token(self.cursor - 1) + .expect("consumed token") + .span + .end, }, )); continue; @@ -1444,20 +1570,40 @@ impl Parser<'_> { if entry_header.description.is_some() { return Err(self.error("Duplicate project description.")); } + self.syntax_entry()?; self.cursor += 1; let description = self.quoted()?; self.take(";")?; entry_header.project.description = description; entry_header.description = Some(Span { start: offset, - end: self.tokens[self.cursor - 1].span.end, + end: self + .token(self.cursor - 1) + .expect("consumed token") + .span + .end, }); header = Some(entry_header); continue; } + #[cfg(test)] + if self.mixed.is_some() && analysis_source::mixed::long_analysis_word(self.text()) { + return Err(self.error("Use the short source8 analysis declaration word.")); + } + self.syntax_entry()?; let word = self.text().to_owned(); let kind = declaration_kind(&word) .ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?; + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.record( + matches!(kind, DeclKind::Element(..) | DeclKind::Relation(..)), + Span { + start: offset, + end: offset, + }, + )?; + } graph_count += usize::from(matches!( kind, DeclKind::Element(..) | DeclKind::Relation(..) @@ -1485,6 +1631,10 @@ impl Parser<'_> { let body_start = self.current().span.end; self.take("{")?; let lexical_owner = open.last().map(|&owner| declarations[owner].id); + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.lexical_owners.push(open.last().copied()); + } declarations.push(Declaration { kind, alias, diff --git a/crates/oa-core/src/language/analysis_source.rs b/crates/oa-core/src/language/analysis_source.rs index dec41849..8f035a5c 100644 --- a/crates/oa-core/src/language/analysis_source.rs +++ b/crates/oa-core/src/language/analysis_source.rs @@ -6,6 +6,7 @@ mod declarations; mod index; mod lex; mod lower; +pub(super) mod mixed; mod syntax; mod update; mod write; diff --git a/crates/oa-core/src/language/analysis_source/mixed.rs b/crates/oa-core/src/language/analysis_source/mixed.rs new file mode 100644 index 00000000..312efcbf --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/mixed.rs @@ -0,0 +1,394 @@ +//! Private syntax-only complete-bundle witness. No model admission or edit authority. +//! +//! One scan charges original bytes and raw tokens (including trivia). Typed +//! syntax keeps its existing entry/reference/numeric charges. Ordinary syntax +//! charges one entry per project/declaration/include/description, field, and +//! value before allocation; array values already account for their list slots. +//! These private storage ceilings are independent of caller model record counts. +//! Ordinary references consume value storage, not ordered-model-step allowance. +//! Typed reference occurrences retain the caller max_step_references ceiling. +//! Only declaration graph/record counts are known here; semantic nested-model +//! records and ordinary ordered steps still require the later complete admission. +//! Metadata phase slots aggregate across all files: retained token-end indices, +//! value-end entries, projected value/field/header/declaration positions, lexical +//! owner slots and the file table. Checks do not recharge syntax or latch errors; +//! every failure must return immediately. Tokens themselves use the token bound. +//! This deliberately bounded staging policy is not calibrated public source8. + +use super::{ + budget::{Budget, Limits}, + cursor::Cursor, + declarations, lex, +}; +use crate::Id; +use crate::language::{ + self, Ast, EditDeclaration, Error, Lexeme, ParseLimits, Parser, SourceUnit, Span, TokenClass, +}; +use crate::source_project::{ProjectLimits, SourceBundle, SourceDiagnostic, traversal::Traversal}; +use std::collections::BTreeMap; + +struct ModelCounts { + limits: ProjectLimits, + graph: usize, + records: usize, + includes: usize, +} + +pub(super) struct ParsedMixedSyntax<'s> { + bundle: &'s SourceBundle, + files: Vec>, + budget: Budget, +} + +struct FileSyntax<'s> { + path: &'s str, + source: &'s str, + tokens: Vec, + ordinary: SourceUnit, + typed: Vec, + positions: Vec<(Id, EditDeclaration)>, + // A lexical owner is an occurrence index, not only a possibly repeated UUID. + lexical_owners: Vec>, +} + +// Only Parser borrows this state; the stored witness has no self-reference. +pub(in crate::language) struct State<'a> { + source: &'a str, + tokens: &'a [lex::Token], + budget: &'a mut Budget, + model: &'a mut ModelCounts, + typed: Vec, + pub(in crate::language) value_ends: BTreeMap, + pub(in crate::language) lexical_owners: Vec>, +} + +fn ordinary_error(error: super::Error) -> Error { + Error { + code: "invalid_source", + offset: error.span.start, + message: error.message, + } +} + +fn lexeme(token: lex::Token) -> Lexeme { + Lexeme { + class: match token.class { + lex::Class::Word => TokenClass::Identifier, + lex::Class::Number => TokenClass::Number, + lex::Class::String => TokenClass::String, + lex::Class::Punctuation => TokenClass::Punctuation, + }, + span: token.span, + } +} + +impl State<'_> { + pub(in crate::language) fn token(&self, at: usize) -> Option { + self.tokens.get(at).copied().map(lexeme) + } + pub(in crate::language) fn entry(&mut self, at: Span) -> Result<(), Error> { + self.budget.entry(at).map_err(ordinary_error) + } + pub(in crate::language) fn record(&mut self, graph: bool, at: Span) -> Result<(), Error> { + let records = self.model.records.checked_add(1); + let graph = self.model.graph.checked_add(usize::from(graph)); + let (Some(records), Some(graph)) = (records, graph) else { + return Err(Error { + code: "model_limit", + offset: at.start, + message: "Declaration count overflow.".into(), + }); + }; + self.model + .limits + .model + .check_counts(graph, records, 0) + .map_err(|message| Error { + code: "model_limit", + offset: at.start, + message, + })?; + self.model.records = records; + self.model.graph = graph; + Ok(()) + } + pub(in crate::language) fn include(&mut self, at: Span) -> Result<(), Error> { + if self.model.includes >= self.model.limits.max_include_edges { + return Err(Error { + code: "project_limit", + offset: at.start, + message: format!( + "Include graph exceeds {} edges.", + self.model.limits.max_include_edges + ), + }); + } + self.model.includes += 1; + Ok(()) + } + pub(in crate::language) fn ordinary_number( + &self, + at: &mut usize, + ) -> Result { + let first = self.tokens[*at]; + let mut end = first.span.end; + let mut after = *at + 1; + // Match the old greedy numeric alphabet only across byte-adjacent raw + // tokens. This is consumption of the shared stream, not another scan. + while let Some(next) = self.tokens.get(after) { + if next.span.start != end + || !self.source[next.span.start..next.span.end] + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.+-".contains(&byte)) + { + break; + } + end = next.span.end; + after += 1; + } + let value: serde_json::Number = serde_json::from_str(&self.source[first.span.start..end]) + .map_err(|_| Error { + code: "invalid_source", + offset: first.span.start, + message: "Expected a byte-adjacent finite JSON decimal number.".into(), + })?; + *at = after; + Ok(serde_json::Value::Number(value)) + } + pub(in crate::language) fn typed_header(&self, at: usize, root: bool) -> bool { + let Some(token) = self.tokens.get(at) else { + return false; + }; + if declarations::declaration_kind(&self.source[token.span.start..token.span.end]).is_none() + { + return false; + } + // A generic field named `unit` remains an ordinary field. A declaration + // header has an alias and display-name token; only that shape is nested. + root || (self + .tokens + .get(at + 1) + .is_some_and(|t| t.class == lex::Class::Word) + && self + .tokens + .get(at + 2) + .is_some_and(|t| t.class == lex::Class::String)) + } + pub(in crate::language) fn parse_typed( + &mut self, + at: &mut usize, + owner: Option, + ) -> Result<(), Error> { + // Transfer the one active position into the typed consumer and back; + // neither consumer scans, clones a stream, nor runs concurrently. + self.record(true, self.tokens[*at].span)?; + let mut cursor = Cursor::new(self.source, self.tokens, self.budget); + cursor.cursor = *at; + let result = declarations::parse(&mut cursor, 8, owner); + *at = cursor.cursor; + let declaration = result.map_err(ordinary_error)?; + let missing = declaration.header.identity.value.is_none() + || matches!(&declaration.body, declarations::Body::Calculation { parameters, .. } + if parameters.iter().any(|p| p.identity.value.is_none())); + if missing { + return Err(Error { + code: "invalid_source", + offset: declaration.span.start, + message: "Private mixed syntax requires explicit IDs.".into(), + }); + } + self.typed.push(declaration); + Ok(()) + } +} + +pub(in crate::language) fn long_analysis_word(word: &str) -> bool { + matches!( + word, + "oa_engineering_property" + | "sa_engineering_property" + | "oa_analysis_check" + | "sa_analysis_check" + | "oa_analysis_case" + | "sa_analysis_case" + ) +} + +pub(super) fn parse( + bundle: &SourceBundle, + limits: ProjectLimits, +) -> Result, Vec> { + let profile = Limits::source8(ParseLimits { + // Storage is not the caller's final model record/ordered-step accounting. + max_source_bytes: limits.model.max_source_bytes, + max_step_references: limits.model.max_step_references, + ..ParseLimits::UNLIMITED + }); + parse_with_profile(bundle, limits, profile) +} + +fn located(path: &str, source: &str, error: Error) -> Vec { + vec![SourceDiagnostic { + path: path.into(), + diagnostic: language::diagnostic(source, error), + related: Vec::new(), + }] +} + +fn parse_with_profile( + bundle: &SourceBundle, + limits: ProjectLimits, + profile: Limits, +) -> Result, Vec> { + let entry_source = bundle.files.get(&bundle.entry).map_or("", String::as_str); + let at_entry = |error| located(&bundle.entry, entry_source, ordinary_error(error)); + let maximum = Limits::source8(ParseLimits { + max_source_bytes: limits.model.max_source_bytes, + max_step_references: limits.model.max_step_references, + ..ParseLimits::UNLIMITED + }); + if profile.bytes > maximum.bytes + || profile.tokens > maximum.tokens + || profile.entries > maximum.entries + || profile.references > maximum.references + || profile.frames > maximum.frames + || profile.numeric_units > maximum.numeric_units + { + return Err(at_entry(super::error( + Span::default(), + "Mixed test profile may only tighten inherited limits.", + ))); + } + let mut budget = Budget::new(profile).map_err(at_entry)?; + let mut traversal = Traversal::new(bundle, limits)?; + let bytes = bundle + .files + .values() + .try_fold(0usize, |n, s| n.checked_add(s.len())) + .ok_or_else(|| at_entry(super::error(Span::default(), "Source size overflow.")))?; + budget + .index_bytes(bytes, Span::default()) + .map_err(at_entry)?; + let mut metadata = bundle.files.len(); + budget + .lowering_entries(metadata, Span::default()) + .map_err(at_entry)?; + let mut parsed = BTreeMap::new(); + let mut model = ModelCounts { + limits, + graph: 0, + records: 0, + includes: 0, + }; + while let Some(path) = traversal.next()? { + let (path, source) = bundle + .files + .get_key_value(&path) + .expect("traversal checked source"); + let failure = |error| located(path, source, ordinary_error(error)); + let tokens = lex::tokens(source, &mut budget).map_err(failure)?; + let mut parser = Parser { + source, + tokens: Vec::new(), + cursor: 0, + value_depth: 0, + limits: limits.model, + mixed: Some(State { + source, + tokens: &tokens, + budget: &mut budget, + model: &mut model, + typed: Vec::new(), + value_ends: BTreeMap::new(), + lexical_owners: Vec::new(), + }), + }; + let ordinary = parser + .unit(path == &bundle.entry, true) + .map_err(|e| located(path, source, e))?; + let state = parser.mixed.take().expect("private parser state"); + drop(parser); + // Projected ordinary AST nodes/fields are bounded before constructing a + // second position representation. Count references as slots, not syntax. + let projection_slots = ordinary_position_slots(&ordinary); + metadata = metadata + .checked_add(tokens.len()) + .and_then(|n| n.checked_add(state.value_ends.len())) + .and_then(|n| n.checked_add(projection_slots)) + .and_then(|n| n.checked_add(state.lexical_owners.len())) + .ok_or_else(|| failure(super::error(Span::default(), "Metadata size overflow.")))?; + state + .budget + .lowering_entries(metadata, Span::default()) + .map_err(failure)?; + let positions = language::edit_positions_from_tokens( + source, + &ordinary, + |at| lexeme(tokens[at]), + tokens.len(), + Some(&state.value_ends), + ) + .map_err(|e| { + vec![SourceDiagnostic { + path: path.clone(), + diagnostic: *e, + related: Vec::new(), + }] + })?; + for (raw, span) in &ordinary.includes { + traversal.include(path, raw, *span)?; + } + traversal.finish_file(path.clone()); + let State { + typed, + lexical_owners, + .. + } = state; + parsed.insert( + path.as_str(), + FileSyntax { + path, + source, + tokens, + ordinary, + typed, + positions, + lexical_owners, + }, + ); + } + traversal.finish()?; + budget.active().map_err(at_entry)?; + Ok(ParsedMixedSyntax { + bundle, + files: parsed.into_values().collect(), + budget, + }) +} + +fn ordinary_position_slots(unit: &SourceUnit) -> usize { + // Iterative accounting avoids adding a second recursive walk before the + // already depth-bounded position projection. Each reference is one value. + let mut slots = 0usize; + let mut stack = Vec::new(); + for declaration in &unit.declarations { + slots = slots.saturating_add(7); // declaration/body + at most five header roles + slots = slots.saturating_add(declaration.fields.len()); + stack.extend(declaration.fields.values()); + } + while let Some(value) = stack.pop() { + slots = slots.saturating_add(1); + match value { + Ast::Array(items, _) => stack.extend(items), + Ast::Object(fields, _) => { + slots = slots.saturating_add(fields.len()); + stack.extend(fields.values()); + } + _ => {} + } + } + slots +} + +#[cfg(test)] +mod tests; diff --git a/crates/oa-core/src/language/analysis_source/mixed/tests.rs b/crates/oa-core/src/language/analysis_source/mixed/tests.rs new file mode 100644 index 00000000..bbb66169 --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/mixed/tests.rs @@ -0,0 +1,921 @@ +use super::*; +use crate::language::{EditKind, EditValue, ParseLimits, Span}; +use crate::source_project::{ProjectLimits, SourceBundle}; +use std::collections::BTreeMap; + +const PROJECT_ID: &str = "00000000-0000-0000-0000-000000000001"; +const FIRST_ID: &str = "00000000-0000-0000-0000-000000000010"; +const SECOND_ID: &str = "00000000-0000-0000-0000-000000000011"; +const CALC_ID: &str = "00000000-0000-0000-0000-000000000100"; +const PARAM_ID: &str = "00000000-0000-0000-0000-000000000200"; + +fn document(body: &str) -> String { + format!("arcadia 8;\nproject p \"Project\" id \"{PROJECT_ID}\" {{\n{body}\n}}\n") +} + +fn single(body: &str) -> SourceBundle { + SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([("main.arcadia".into(), document(body))]), + } +} + +fn calculation(alias: &str, id: &str, expression: &str) -> String { + format!( + "calculation {alias} \"Calculation\" id \"{id}\" {{ result Number; body {expression}; }}" + ) +} + +fn ordinary(alias: &str, id: &str, fields: &str) -> String { + format!("oa_activity {alias} \"Activity\" id \"{id}\" {{ {fields} }}") +} + +fn text(source: &str, span: Span) -> &str { + &source[span.start..span.end] +} + +fn exact_span(source: &str, authored: &str) -> Span { + // The expected substring is authored here, independently of parser output. + let start = source.find(authored).expect("unique authored fixture text"); + assert_eq!(source.matches(authored).count(), 1); + Span { + start, + end: start + authored.len(), + } +} + +fn fields(value: &EditValue) -> &BTreeMap { + let EditKind::Record(fields) = &value.kind else { + panic!("record position") + }; + fields +} + +fn rejects(bundle: &SourceBundle) -> Vec { + let before = bundle.clone(); + let errors = match parse(bundle, ProjectLimits::default()) { + Ok(_) => panic!("invalid syntax returned a complete mixed witness"), + Err(errors) => errors, + }; + assert!(!errors.is_empty()); + assert_eq!(bundle, &before, "failure may not stabilize or mutate input"); + errors +} + +#[test] +fn interleaved_unicode_crlf_has_original_typed_and_ordinary_coordinates() { + let typed = format!("calculation c \"数値\" id \"{CALC_ID}\" {{ result Number; body 1-2; }}"); + let child = format!("oa_activity child \"子\" id \"{SECOND_ID}\" {{ description \"雪 @c\"; }}"); + let parent = format!("oa_activity container \"親\" id \"{FIRST_ID}\" {{\r\n {child}\r\n}}"); + let source = format!( + "// 雪 before every coordinate\r\narcadia 8;\r\nproject p \"港\" id \"{PROJECT_ID}\" {{\r\n description \"計画\";\r\n {parent}\r\n {typed}\r\n include \"lib.arcadia\";\r\n}}\r\n" + ); + let library = ordinary("later", PARAM_ID, "description \"é\";"); + let bundle = SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([ + ("main.arcadia".into(), source.clone()), + ("lib.arcadia".into(), library), + ]), + }; + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + assert!(std::ptr::eq(parsed.bundle, &bundle)); + assert_eq!( + parsed.files.iter().map(|f| f.path).collect::>(), + ["lib.arcadia", "main.arcadia"] + ); + let file = &parsed.files[1]; + assert!(std::ptr::eq( + file.source.as_ptr(), + bundle.files["main.arcadia"].as_ptr() + )); + assert_eq!(file.source.len(), source.len()); + assert_eq!(file.ordinary.declarations.len(), 2); + assert_eq!(file.typed.len(), 1); + assert_eq!(file.positions.len(), 2); + assert_eq!(file.lexical_owners, [None, Some(0)]); + assert_eq!( + file.ordinary.declarations[0].span, + exact_span(&source, &parent) + ); + assert_eq!( + file.ordinary.declarations[1].span, + exact_span(&source, &child) + ); + assert_eq!(file.positions[1].1.span, exact_span(&source, &child)); + assert_eq!(file.typed[0].span, exact_span(&source, &typed)); + let description = &fields(&file.positions[1].1.fields)["description"]; + assert_eq!( + description.span, + exact_span(&source, "description \"雪 @c\";") + ); + assert_eq!(description.value.span, exact_span(&source, "\"雪 @c\"")); + assert_eq!( + file.ordinary.includes[0].1, + exact_span(&source, "include \"lib.arcadia\";") + ); + assert_eq!( + text( + &source, + file.ordinary.header.as_ref().unwrap().description.unwrap() + ), + "description \"計画\";" + ); + assert_eq!( + parsed.budget.usage.bytes, + bundle.files.values().map(String::len).sum::() + ); + assert!( + file.tokens + .windows(2) + .all(|pair| pair[0].span.end <= pair[1].span.start) + ); + assert!( + file.tokens + .iter() + .all(|token| source.is_char_boundary(token.span.start) + && source.is_char_boundary(token.span.end)) + ); + let declarations::Body::Calculation { body, .. } = &file.typed[0].body else { + panic!("calculation") + }; + assert_eq!(text(&source, body.value_span), "1-2"); + let super::super::Binding::Expression(expression) = &body.value else { + panic!("expression") + }; + assert!(matches!( + &expression.kind, + super::super::ExpressionKind::Binary { + op: crate::analysis::BinaryOp::Subtract, + .. + } + )); +} + +#[test] +fn duplicate_ordinary_ids_keep_occurrences_and_scoped_parameter_ids_remain_distinct() { + let child_a = ordinary("child_a", SECOND_ID, ""); + let child_b = ordinary("child_b", SECOND_ID, ""); + let parent_a = ordinary("parent_a", FIRST_ID, &child_a); + let parent_b = ordinary("parent_b", FIRST_ID, &child_b); + let calc = |alias: &str, id: &str| { + format!( + "calculation {alias} \"Scoped\" id \"{id}\" {{ input p id \"{PARAM_ID}\" : Number; result Number; body $p; }}" + ) + }; + let bundle = single(&format!( + "{parent_a}\n{}\n{parent_b}\n{}", + calc("f", CALC_ID), + calc("g", PARAM_ID) + )); + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + let file = &parsed.files[0]; + assert_eq!( + file.ordinary + .declarations + .iter() + .map(|d| d.alias.as_str()) + .collect::>(), + ["parent_a", "child_a", "parent_b", "child_b"] + ); + assert_eq!(file.lexical_owners, [None, Some(0), None, Some(2)]); + assert_eq!( + file.positions.iter().map(|(id, _)| *id).collect::>(), + [FIRST_ID, SECOND_ID, FIRST_ID, SECOND_ID].map(|id| Id::parse_str(id).unwrap()) + ); + for (index, authored) in [&parent_a, &child_a, &parent_b, &child_b] + .into_iter() + .enumerate() + { + assert_eq!( + file.positions[index].1.span, + exact_span(file.source, authored) + ); + } + assert_eq!(file.typed.len(), 2); + for declaration in &file.typed { + let declarations::Body::Calculation { + parameters, body, .. + } = &declaration.body + else { + panic!("calculation") + }; + assert_eq!(parameters.len(), 1); + assert_eq!( + parameters[0].identity.value, + Some(Id::parse_str(PARAM_ID).unwrap()) + ); + assert_eq!(parameters[0].name.name, "p"); + assert_eq!(text(file.source, body.value_span), "$p"); + } + // Syntax deliberately does not establish global identity uniqueness. + assert_eq!( + file.typed[1].header.identity.value, + Some(Id::parse_str(PARAM_ID).unwrap()) + ); +} + +#[test] +fn diamond_includes_return_one_canonical_file_table_and_scan_dependency_once() { + let bundle = SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([ + ( + "main.arcadia".into(), + document("include \"root/z.arcadia\"; include \"root/a.arcadia\";"), + ), + ( + "root/z.arcadia".into(), + "include \"../shared.arcadia\";".into(), + ), + ( + "root/a.arcadia".into(), + "include \"../shared.arcadia\";".into(), + ), + ( + "shared.arcadia".into(), + calculation("shared", CALC_ID, "1e-2"), + ), + ]), + }; + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + assert_eq!( + parsed.files.iter().map(|f| f.path).collect::>(), + [ + "main.arcadia", + "root/a.arcadia", + "root/z.arcadia", + "shared.arcadia" + ] + ); + assert_eq!(parsed.files.iter().map(|f| f.typed.len()).sum::(), 1); + assert_eq!( + parsed + .files + .iter() + .map(|f| f.ordinary.includes.len()) + .sum::(), + 4 + ); + assert_eq!( + parsed.budget.usage.bytes, + bundle.files.values().map(String::len).sum::() + ); + assert_eq!( + parsed + .files + .iter() + .filter(|f| f.ordinary.header.is_some()) + .count(), + 1 + ); + assert!(parsed.files[0].ordinary.header.is_some()); + let limits = ProjectLimits { + max_files: 4, + max_include_depth: 2, + max_include_edges: 4, + ..ProjectLimits::default() + }; + assert!(parse(&bundle, limits).is_ok()); + for limits in [ + ProjectLimits { + max_files: 3, + ..limits + }, + ProjectLimits { + max_include_depth: 1, + ..limits + }, + ProjectLimits { + max_include_edges: 3, + ..limits + }, + ] { + assert!(parse(&bundle, limits).is_err()); + } +} + +#[test] +fn whole_bundle_rejects_missing_unreachable_cyclic_and_noncanonical_files() { + let mut missing = single("include \"missing.arcadia\";"); + rejects(&missing); + missing.entry = "absent.arcadia".into(); + rejects(&missing); + let mut extra = single(""); + extra.files.insert( + "orphan.arcadia".into(), + "// untouched but unreachable".into(), + ); + rejects(&extra); + let mut cycle = single("include \"child.arcadia\";"); + cycle + .files + .insert("child.arcadia".into(), "include \"main.arcadia\";".into()); + rejects(&cycle); + let mut collision = single("include \"MAIN.arcadia\";"); + collision.files.insert("MAIN.arcadia".into(), "".into()); + rejects(&collision); + for key in [ + "./child.arcadia", + "x/../child.arcadia", + "dir\\child.arcadia", + "/child.arcadia", + ] { + let mut bundle = single(""); + bundle.files.insert(key.into(), "".into()); + rejects(&bundle); + } + for include in [ + "../outside.arcadia", + "/absolute.arcadia", + "x\\child.arcadia", + ] { + // JSON quoting deliberately makes the backslash an actual path byte. + rejects(&single(&format!( + "include {};", + serde_json::to_string(include).unwrap() + ))); + } +} + +#[test] +fn ordinary_signed_numbers_keep_complete_nested_value_and_field_ends() { + let body = ordinary( + "a", + FIRST_ID, + "weight -12.5; nested { value -1e-2; items [-3, { tail -4.5; }]; }; parent @target;", + ); + let bundle = single(&body); + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + let file = &parsed.files[0]; + let top = fields(&file.positions[0].1.fields); + for (key, value, field) in [ + ("weight", "-12.5", "weight -12.5;"), + ( + "nested", + "{ value -1e-2; items [-3, { tail -4.5; }]; }", + "nested { value -1e-2; items [-3, { tail -4.5; }]; };", + ), + ("parent", "@target", "parent @target;"), + ] { + assert_eq!(top[key].value.span, exact_span(file.source, value)); + assert_eq!(top[key].span, exact_span(file.source, field)); + assert_eq!( + file.ordinary.declarations[0].field_spans[key], + exact_span(file.source, value) + ); + } + let nested = fields(&top["nested"].value); + assert_eq!(nested["value"].value.span, exact_span(file.source, "-1e-2")); + assert_eq!( + nested["value"].span, + exact_span(file.source, "value -1e-2;") + ); + assert_eq!( + nested["items"].span, + exact_span(file.source, "items [-3, { tail -4.5; }];") + ); + let EditKind::List(items) = &nested["items"].value.kind else { + panic!("list") + }; + assert_eq!(items.len(), 2); + assert_eq!(items[0].span, exact_span(file.source, "-3")); + assert_eq!(items[1].span, exact_span(file.source, "{ tail -4.5; }")); + assert_eq!( + fields(&items[1])["tail"].span, + exact_span(file.source, "tail -4.5;") + ); + assert_eq!( + fields(&items[1])["tail"].value.span, + exact_span(file.source, "-4.5") + ); + // Equal-length version substitution retains all byte coordinates. These + // comparisons supplement, rather than generate, the authored oracles above. + let legacy = file.source.replacen("arcadia 8;", "arcadia 7;", 1); + let legacy_positions = + crate::language::edit_declarations(&legacy, true, ParseLimits::UNLIMITED).unwrap(); + let old = fields(&legacy_positions[0].1.fields); + assert_eq!(old["weight"].value.span, top["weight"].value.span); + assert_eq!(old["nested"].span, top["nested"].span); + assert_eq!( + fields(&old["nested"].value)["items"].span, + nested["items"].span + ); +} + +#[test] +fn ordinary_numeric_adjacency_and_malformed_trivia_match_legacy_refusals() { + for value in [ + "- 12.5", + "- // comment\n12.5", + "+1", + ".5", + "1e", + "1_0", + "0x1", + "1-2", + "1e9999", + "@ /*not supported*/ target", + ] { + for field in [ + format!("weight {value};"), + format!("nested {{ items [{value}]; }};"), + ] { + let bundle = single(&ordinary("a", FIRST_ID, &field)); + rejects(&bundle); + let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1); + assert!( + crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_err(), + "legacy unexpectedly accepts {value}" + ); + } + } + for value in ["-12.5", "-1e-2", "1e-2", "0", "-0", "1E+2"] { + let bundle = single(&ordinary("a", FIRST_ID, &format!("weight {value};"))); + assert!( + parse(&bundle, ProjectLimits::default()).is_ok(), + "mixed refuses {value}" + ); + let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1); + assert!( + crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_ok(), + "legacy refuses {value}" + ); + } +} + +#[test] +fn typed_words_are_contextual_but_typed_headers_cannot_nest_in_ordinary_bodies() { + let bundle = single(&format!( + "{}\n{}", + ordinary( + "unit", + FIRST_ID, + "unit 1; calculation \"word\"; quantity_kind none;" + ), + calculation("calculation", CALC_ID, "1") + )); + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + let file = &parsed.files[0]; + assert_eq!(file.ordinary.declarations[0].alias, "unit"); + assert_eq!(file.typed[0].header.alias.name, "calculation"); + assert_eq!( + fields(&file.positions[0].1.fields) + .keys() + .map(String::as_str) + .collect::>(), + ["calculation", "quantity_kind", "unit"] + ); + rejects(&single(&ordinary( + "a", + FIRST_ID, + &calculation("c", CALC_ID, "1"), + ))); + for word in [ + "oa_engineering_property", + "sa_engineering_property", + "oa_analysis_check", + "sa_analysis_check", + "oa_analysis_case", + "sa_analysis_case", + ] { + let forbidden = format!("{word} a \"Long derived kind\" id \"{FIRST_ID}\" {{}}"); + rejects(&single(&forbidden)); + rejects(&single(&ordinary("owner", SECOND_ID, &forbidden))); + } +} + +#[test] +fn every_identity_class_requires_authored_ids_without_mutating_the_bundle() { + let project_missing = SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([( + "main.arcadia".into(), + "arcadia 8; project p \"Project\" {}".into(), + )]), + }; + rejects(&project_missing); + for body in [ + "oa_activity a \"Missing\" {}".to_owned(), + ordinary("owner", FIRST_ID, "oa_activity child \"Missing\" {}"), + "calculation c \"Missing\" { result Number; body 1; }".to_owned(), + format!( + "calculation c \"Missing parameter\" id \"{CALC_ID}\" {{ input p : Number; result Number; body $p; }}" + ), + "quantity_kind q \"Missing\" { dimension {}; algebra Linear; }".to_owned(), + "unit u \"Missing\" { kind @q; scale 1; offset 0; symbol \"u\"; }".to_owned(), + ] { + rejects(&single(&body)); + } + let mut included = single("include \"lib.arcadia\";"); + included + .files + .insert("lib.arcadia".into(), "oa_activity a \"Missing\" {}".into()); + rejects(&included); +} + +#[test] +fn ordinary_value_depth_boundary_remains_64_in_both_parser_views() { + // A scalar consumes the innermost level: 63 containing lists + scalar = 64. + for (lists, accepted) in [(63, true), (64, false)] { + let value = format!("{}1{}", "[".repeat(lists), "]".repeat(lists)); + let bundle = single(&ordinary("a", FIRST_ID, &format!("values {value};"))); + assert_eq!(parse(&bundle, ProjectLimits::default()).is_ok(), accepted); + let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1); + assert_eq!( + crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_ok(), + accepted + ); + } +} + +#[test] +fn exact_shared_profile_limits_include_nonlatching_metadata_phase_failures() { + // Independent hand count: 10 significant tokens and 5 intervening spaces. + // No declaration positions, value-end entries, or lexical owners exist. + // The phase table therefore needs 1 file slot + 10 token-end slots = 11. + let source = format!("arcadia 8;project p \"P\" id \"{PROJECT_ID}\"{{}}"); + let bundle = SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([("main.arcadia".into(), source.clone())]), + }; + let limits = ProjectLimits { + model: ParseLimits { + max_records: 1, + max_step_references: 0, + max_source_bytes: source.len(), + ..ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }; + let profile = Limits { + bytes: source.len(), + tokens: 15, + entries: 11, + references: 0, + numeric_units: 0, + ..Limits::source8(ParseLimits::UNLIMITED) + }; + let parsed = parse_with_profile(&bundle, limits, profile).unwrap(); + assert_eq!(parsed.files.len(), 1); + assert_eq!(parsed.files[0].tokens.len(), 10); + assert_eq!(parsed.budget.usage.bytes, source.len()); + assert_eq!(parsed.budget.usage.tokens, 15); + assert_eq!(parsed.budget.usage.entries, 1); + assert_eq!(parsed.budget.usage.references, 0); + assert_eq!(parsed.budget.numeric_units(), 0); + for (profile, message) in [ + ( + Limits { + entries: 10, + ..profile + }, + "Source8 lowering entry limit exceeded.", + ), + ( + Limits { + tokens: 14, + ..profile + }, + "Source8 tokens limit exceeded.", + ), + ( + Limits { + bytes: source.len() - 1, + ..profile + }, + "Source8 index/output byte limit exceeded.", + ), + ] { + let errors = match parse_with_profile(&bundle, limits, profile) { + Ok(_) => panic!("exhausted phase returned a complete witness"), + Err(errors) => errors, + }; + assert!( + errors + .iter() + .any(|error| error.diagnostic.message == message) + ); + assert_eq!(bundle.files["main.arcadia"], source); + } + // A caller source-byte limit is inherited even when a test profile would + // otherwise fall within the candidate hard ceiling. + let tighter = ProjectLimits { + model: ParseLimits { + max_source_bytes: source.len() - 1, + ..limits.model + }, + ..limits + }; + assert!(parse_with_profile(&bundle, tighter, profile).is_err()); +} + +#[test] +fn ordinary_model_records_and_parent_reference_are_not_syntax_or_step_limits() { + let bundle = single(&ordinary("a", FIRST_ID, "parent @target;")); + let limits = ProjectLimits { + model: ParseLimits { + max_records: 2, + max_step_references: 0, + ..ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }; + let parsed = parse(&bundle, limits).unwrap(); + assert_eq!(parsed.files[0].ordinary.declarations.len(), 1); + // Project, ordinary declaration, field, and reference value: four storage + // entries, while only project + declaration count as two model records. + assert_eq!(parsed.budget.usage.entries, 4); + assert_eq!(parsed.budget.usage.references, 0); + assert_eq!( + text( + parsed.files[0].source, + fields(&parsed.files[0].positions[0].1.fields)["parent"] + .value + .span + ), + "@target" + ); + assert!( + parse( + &bundle, + ProjectLimits { + model: ParseLimits { + max_records: 1, + ..limits.model + }, + ..limits + } + ) + .is_err() + ); +} + +#[test] +fn typed_numeric_work_remains_in_the_same_inherited_profile() { + let bundle = single(&calculation("c", CALC_ID, "1e-2")); + let limits = ProjectLimits::default(); + let parsed = parse(&bundle, limits).unwrap(); + assert!(parsed.budget.numeric_units() > 0); + let profile = Limits { + numeric_units: 0, + ..Limits::source8(ParseLimits::UNLIMITED) + }; + let before = bundle.clone(); + assert!(parse_with_profile(&bundle, limits, profile).is_err()); + assert_eq!(bundle, before); +} + +#[test] +fn all_short_typed_roots_share_original_buffers_without_public_activation() { + use crate::Layer; + use declarations::DeclarationKind as K; + let rows = [ + ( + "quantity_kind", + "q", + "dimension {}; algebra Linear;", + K::QuantityKind, + ), + ( + "unit", + "u", + "kind @q; scale ratio(1, 10); offset 0; symbol \"u\";", + K::Unit, + ), + ( + "calculation", + "c", + "result Number; body 123456789012345678901234567890123456789;", + K::Calculation, + ), + ( + "oa_property", + "op", + "subject @later; type Number; base unbound;", + K::Property(Layer::Oa), + ), + ( + "sa_property", + "sp", + "subject @later; type Number; base unbound;", + K::Property(Layer::Sa), + ), + ( + "oa_check", + "oc", + "subject none; predicate true;", + K::Check(Layer::Oa), + ), + ( + "sa_check", + "sc", + "subject none; predicate true;", + K::Check(Layer::Sa), + ), + ( + "oa_analysis", + "case_oa", + "subject none; base ModelBaseline; overrides {};", + K::Case(Layer::Oa), + ), + ( + "sa_analysis", + "case_sa", + "subject none; base ModelBaseline; overrides {};", + K::Case(Layer::Sa), + ), + ]; + let declarations: Vec<_> = rows + .iter() + .enumerate() + .map(|(n, (word, alias, body, _))| { + format!( + "{word} {alias} \"Typed\" id \"00000000-0000-0000-0000-{:012}\" {{ {body} }}", + n + 1000 + ) + }) + .collect(); + let bundle = single(&format!( + "{}\n{}", + declarations.join("\n"), + ordinary("later", FIRST_ID, "") + )); + let parsed = parse(&bundle, ProjectLimits::default()).unwrap(); + let file = &parsed.files[0]; + assert_eq!(file.typed.len(), 9); + for ((actual, (_, alias, _, kind)), authored) in file.typed.iter().zip(rows).zip(&declarations) + { + assert_eq!(actual.header.kind, kind); + assert_eq!(actual.header.alias.name, alias); + assert_eq!(actual.span, exact_span(file.source, authored)); + } + let declarations::Body::Calculation { body, .. } = &file.typed[2].body else { + panic!("calculation") + }; + let super::super::Binding::Expression(expression) = &body.value else { + panic!("expression") + }; + let super::super::ExpressionKind::Number(value) = &expression.kind else { + panic!("exact number") + }; + assert_eq!( + value.canonical_literal(), + "123456789012345678901234567890123456789" + ); + // The checked private mode does not change any public source8 entrypoint. + assert!(crate::language::parse_unit(file.source, true, ParseLimits::UNLIMITED).is_err()); + assert!(crate::source_project::compile_bundle(&bundle, ProjectLimits::default()).is_err()); + let old = SourceBundle { + entry: bundle.entry.clone(), + files: BTreeMap::from([( + bundle.entry.clone(), + file.source.replacen("arcadia 8", "arcadia 7", 1), + )]), + }; + rejects(&old); + let mut fragment_header = single("include \"fragment.arcadia\";"); + fragment_header + .files + .insert("fragment.arcadia".into(), document("")); + rejects(&fragment_header); +} + +#[test] +fn mixed_typed_reference_limit_is_inherited_without_charging_ordinary_parent_refs() { + let bundle = single(&format!( + "{}\n{}", + ordinary("a", FIRST_ID, "parent @unresolved;"), + calculation("c", CALC_ID, "@unresolved") + )); + let limits = ProjectLimits { + model: ParseLimits { + max_step_references: 1, + ..ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }; + let parsed = parse(&bundle, limits).unwrap(); + assert_eq!(parsed.budget.usage.references, 1); + assert!( + parse( + &bundle, + ProjectLimits { + model: ParseLimits { + max_step_references: 0, + ..limits.model + }, + ..limits + } + ) + .is_err() + ); +} + +#[test] +fn caller_declaration_counts_are_aggregate_and_checked_before_typed_header_allocation() { + let mut bundle = single(&format!( + "{}\ninclude \"later.arcadia\";", + ordinary("early", FIRST_ID, "") + )); + bundle + .files + .insert("later.arcadia".into(), calculation("later", CALC_ID, "1")); + let limits = ProjectLimits { + model: ParseLimits { + max_records: 3, + max_graph_objects: 2, + ..ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }; + assert!(parse(&bundle, limits).is_ok()); + // The third declaration is deliberately missing its header fields. A tighter + // aggregate record ceiling must refuse at its kind before parsing/allocating + // the alias, name, ID, or body; a fresh per-file counter would miss it. + bundle + .files + .insert("later.arcadia".into(), "calculation".into()); + for model in [ + ParseLimits { + max_records: 2, + ..limits.model + }, + ParseLimits { + max_graph_objects: 1, + ..limits.model + }, + ] { + let errors = match parse(&bundle, ProjectLimits { model, ..limits }) { + Ok(_) => panic!("record limit returned a witness"), + Err(errors) => errors, + }; + assert_eq!(errors[0].path, "later.arcadia"); + assert_eq!(errors[0].diagnostic.code, "model_limit"); + assert_eq!(errors[0].diagnostic.span.start, 0); + } + let includes = single("include true;"); + let errors = match parse( + &includes, + ProjectLimits { + max_include_edges: 0, + ..limits + }, + ) { + Ok(_) => panic!("include limit returned a witness"), + Err(errors) => errors, + }; + assert_eq!(errors[0].diagnostic.code, "project_limit"); + assert!(errors[0].diagnostic.message.contains("Include graph")); +} + +#[test] +fn mixed_deep_typed_success_and_rejection_cleanup_on_128k_stack() { + std::thread::Builder::new() + .name("mixed-syntax-128k".into()) + .stack_size(128 * 1024) + .spawn(|| { + for (wrappers, accepted) in [(127, true), (128, false)] { + let expression = format!("{}1{}", "abs(".repeat(wrappers), ")".repeat(wrappers)); + let bundle = single(&format!( + "{}\n{}\n{}", + ordinary("before", FIRST_ID, ""), + calculation("c", CALC_ID, &expression), + ordinary("after", SECOND_ID, "") + )); + match parse(&bundle, ProjectLimits::default()) { + Ok(witness) => { + assert!(accepted); + let declarations::Body::Calculation { body, .. } = + &witness.files[0].typed[0].body + else { + panic!("calculation") + }; + let super::super::Binding::Expression(expr) = &body.value else { + panic!("expression") + }; + assert_eq!(expr.depth, 128); + drop(witness); + } + Err(errors) => { + assert!(!accepted); + assert!( + errors + .iter() + .any(|e| e.diagnostic.message.contains("depth")) + ); + } + } + } + assert!( + parse( + &single(&calculation("fresh", CALC_ID, "1")), + ProjectLimits::default() + ) + .is_ok() + ); + }) + .unwrap() + .join() + .unwrap(); +} diff --git a/crates/oa-core/src/source_project.rs b/crates/oa-core/src/source_project.rs index 4d3e377f..a2e49991 100644 --- a/crates/oa-core/src/source_project.rs +++ b/crates/oa-core/src/source_project.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; mod source_migration; #[path = "source_project/source_structure.rs"] mod source_structure; +pub(crate) mod traversal; /// A portable slash-separated path, checked at every bundle boundary. pub type LogicalPath = String; @@ -304,71 +305,7 @@ fn compile_once( bundle: &SourceBundle, limits: ProjectLimits, ) -> Result<(CompiledSourceProject, Edits), Vec> { - let fail = |code, message| { - vec![problem( - bundle, - &bundle.entry, - Span::default(), - code, - message, - )] - }; - if bundle.entry.contains('/') { - return Err(fail( - "invalid_path", - "The entry file must be at the source project root.".into(), - )); - } - if bundle.files.len() > limits.max_files { - return Err(fail( - "project_limit", - format!("Source project exceeds {} files.", limits.max_files), - )); - } - let bytes = bundle - .files - .values() - .fold(0usize, |sum, text| sum.saturating_add(text.len())); - if bytes > limits.model.max_source_bytes { - return Err(fail( - "source_limit", - format!( - "Combined source exceeds {} bytes.", - limits.model.max_source_bytes - ), - )); - } - let mut folded = BTreeMap::new(); - for path in std::iter::once(&bundle.entry).chain(bundle.files.keys()) { - if resolve_include("", path).as_ref() != Ok(path) { - return Err(fail( - "invalid_path", - format!("Non-canonical source path '{path}'."), - )); - } - if let Some(previous) = folded.insert(path.to_lowercase(), path) - && previous != path - { - return Err(fail( - "duplicate_path", - format!("Case-colliding source paths '{previous}' and '{path}'."), - )); - } - } - if !bundle.files.contains_key(&bundle.entry) { - return Err(fail( - "missing_include", - "The entry file is missing from the source bundle.".into(), - )); - } - - enum Visit { - Enter(LogicalPath, usize, Option), - Leave, - } - let mut stack = vec![Visit::Enter(bundle.entry.clone(), 0, None)]; - let mut active: Vec = Vec::new(); - let mut visited = BTreeSet::new(); + let mut traversal = traversal::Traversal::new(bundle, limits)?; let mut declarations = Vec::new(); let mut declaration_paths = Vec::new(); let mut aliases = BTreeMap::new(); @@ -381,51 +318,8 @@ fn compile_once( let mut includes = Vec::new(); let mut insertions: Edits = BTreeMap::new(); let mut graph_count = 0usize; - while let Some(visit) = stack.pop() { - let Visit::Enter(path, depth, origin) = visit else { - active.pop(); - continue; - }; - let at = origin.unwrap_or(SourceLocation { - path: path.clone(), - span: Span::default(), - }); - if let Some(start) = active.iter().position(|file| file == &path) { - let chain = active[start..] - .iter() - .chain(std::iter::once(&path)) - .cloned() - .collect::>() - .join(" -> "); - return Err(vec![problem( - bundle, - &at.path, - at.span, - "include_cycle", - format!("Include cycle: {chain}."), - )]); - } - if depth > limits.max_include_depth { - return Err(vec![problem( - bundle, - &at.path, - at.span, - "project_limit", - format!("Include depth exceeds {}.", limits.max_include_depth), - )]); - } - if !visited.insert(path.clone()) { - continue; - } - let Some(source) = bundle.files.get(&path) else { - return Err(vec![problem( - bundle, - &at.path, - at.span, - "missing_include", - format!("Included file '{path}' is missing."), - )]); - }; + while let Some(path) = traversal.next()? { + let source = &bundle.files[&path]; let mut unit = language::parse_unit(source, path == bundle.entry, limits.model).map_err( |diagnostic| { vec![SourceDiagnostic { @@ -513,46 +407,12 @@ fn compile_once( declarations.push(declaration); declaration_paths.push(path.clone()); } - let mut children = Vec::new(); for (raw, span) in unit.includes { - let target = resolve_include(&path, &raw) - .map_err(|message| vec![problem(bundle, &path, span, "invalid_path", message)])?; - let location = SourceLocation { - path: path.clone(), - span, - }; - includes.push(IncludeLocation { - location: location.clone(), - target: target.clone(), - }); - if includes.len() > limits.max_include_edges { - return Err(vec![problem( - bundle, - &path, - span, - "project_limit", - format!("Include graph exceeds {} edges.", limits.max_include_edges), - )]); - } - children.push(Visit::Enter( - target, - depth.saturating_add(1), - Some(location), - )); + includes.push(traversal.include(&path, &raw, span)?); } - active.push(path); - stack.push(Visit::Leave); - stack.extend(children.into_iter().rev()); - } - if let Some(path) = bundle.files.keys().find(|path| !visited.contains(*path)) { - return Err(vec![problem( - bundle, - path, - Span::default(), - "unreachable_file", - "Supplied source file is not reachable from the entry file.", - )]); + traversal.finish_file(path); } + traversal.finish()?; let header = project_header.expect("validated entry header"); let index = SourceIndex { declarations: declaration_locations, diff --git a/crates/oa-core/src/source_project/traversal.rs b/crates/oa-core/src/source_project/traversal.rs new file mode 100644 index 00000000..951f71db --- /dev/null +++ b/crates/oa-core/src/source_project/traversal.rs @@ -0,0 +1,258 @@ +//! Shared include traversal policy for ordinary compilation and private syntax. +use super::*; + +enum Visit { + Enter(LogicalPath, usize, Option), + Leave, +} + +pub(crate) struct Traversal<'a> { + bundle: &'a SourceBundle, + limits: ProjectLimits, + stack: Vec, + active: Vec, + visited: BTreeSet, + children: Vec, + depth: usize, + edges: usize, +} + +impl<'a> Traversal<'a> { + pub(crate) fn new( + bundle: &'a SourceBundle, + limits: ProjectLimits, + ) -> Result> { + let fail = |code, message| { + vec![problem( + bundle, + &bundle.entry, + Span::default(), + code, + message, + )] + }; + if bundle.entry.contains('/') { + return Err(fail( + "invalid_path", + "The entry file must be at the source project root.".into(), + )); + } + if bundle.files.len() > limits.max_files { + return Err(fail( + "project_limit", + format!("Source project exceeds {} files.", limits.max_files), + )); + } + let bytes = bundle + .files + .values() + .fold(0usize, |sum, text| sum.saturating_add(text.len())); + if bytes > limits.model.max_source_bytes { + return Err(fail( + "source_limit", + format!( + "Combined source exceeds {} bytes.", + limits.model.max_source_bytes + ), + )); + } + let mut folded = BTreeMap::new(); + for path in std::iter::once(&bundle.entry).chain(bundle.files.keys()) { + if resolve_include("", path).as_ref() != Ok(path) { + return Err(fail( + "invalid_path", + format!("Non-canonical source path '{path}'."), + )); + } + if let Some(previous) = folded.insert(path.to_lowercase(), path) + && previous != path + { + return Err(fail( + "duplicate_path", + format!("Case-colliding source paths '{previous}' and '{path}'."), + )); + } + } + if !bundle.files.contains_key(&bundle.entry) { + return Err(fail( + "missing_include", + "The entry file is missing from the source bundle.".into(), + )); + } + Ok(Self { + bundle, + limits, + stack: vec![Visit::Enter(bundle.entry.clone(), 0, None)], + active: Vec::new(), + visited: BTreeSet::new(), + children: Vec::new(), + depth: 0, + edges: 0, + }) + } + + pub(crate) fn next(&mut self) -> Result, Vec> { + while let Some(visit) = self.stack.pop() { + let Visit::Enter(path, depth, origin) = visit else { + self.active.pop(); + continue; + }; + let at = origin.unwrap_or(SourceLocation { + path: path.clone(), + span: Span::default(), + }); + if let Some(start) = self.active.iter().position(|file| file == &path) { + let chain = self.active[start..] + .iter() + .chain(std::iter::once(&path)) + .cloned() + .collect::>() + .join(" -> "); + return Err(vec![problem( + self.bundle, + &at.path, + at.span, + "include_cycle", + format!("Include cycle: {chain}."), + )]); + } + if depth > self.limits.max_include_depth { + return Err(vec![problem( + self.bundle, + &at.path, + at.span, + "project_limit", + format!("Include depth exceeds {}.", self.limits.max_include_depth), + )]); + } + if !self.visited.insert(path.clone()) { + continue; + } + if !self.bundle.files.contains_key(&path) { + return Err(vec![problem( + self.bundle, + &at.path, + at.span, + "missing_include", + format!("Included file '{path}' is missing."), + )]); + } + self.depth = depth; + return Ok(Some(path)); + } + Ok(None) + } + + pub(crate) fn include( + &mut self, + path: &str, + raw: &str, + span: Span, + ) -> Result> { + let target = resolve_include(path, raw) + .map_err(|message| vec![problem(self.bundle, path, span, "invalid_path", message)])?; + let location = SourceLocation { + path: path.into(), + span, + }; + if self.edges >= self.limits.max_include_edges { + return Err(vec![problem( + self.bundle, + path, + span, + "project_limit", + format!( + "Include graph exceeds {} edges.", + self.limits.max_include_edges + ), + )]); + } + self.edges += 1; + self.children.push(Visit::Enter( + target.clone(), + self.depth.saturating_add(1), + Some(location.clone()), + )); + Ok(IncludeLocation { location, target }) + } + + pub(crate) fn finish_file(&mut self, path: LogicalPath) { + self.active.push(path); + self.stack.push(Visit::Leave); + self.stack.extend(self.children.drain(..).rev()); + } + + pub(crate) fn finish(self) -> Result<(), Vec> { + if let Some(path) = self + .bundle + .files + .keys() + .find(|path| !self.visited.contains(*path)) + { + return Err(vec![problem( + self.bundle, + path, + Span::default(), + "unreachable_file", + "Supplied source file is not reachable from the entry file.", + )]); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ordinary_compile_once_keeps_include_order_and_cycle_source_location() { + let mut bundle = SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([ + ("main.arcadia".into(), "arcadia 7; project p \"P\" id \"00000000-0000-0000-0000-000000000001\" { include \"dir/left.arcadia\"; include \"dir/right.arcadia\"; }".into()), + ("dir/left.arcadia".into(), "include \"../shared.arcadia\";".into()), + ("dir/right.arcadia".into(), "include \"../shared.arcadia\";".into()), + ("shared.arcadia".into(), "// one shared ordinary fragment\n".into()), + ]), + }; + let original = bundle.clone(); + // This is the existing initial ordinary pass, before schema migration's + // separate output/reparse. It makes no public source8 acceptance claim. + let (compiled, insertions) = + super::super::compile_once(&bundle, ProjectLimits::default()).unwrap(); + assert!(insertions.is_empty()); + assert_eq!(compiled.bundle, original); + assert_eq!(compiled.index.declarations.len(), 1); + assert_eq!( + compiled + .index + .includes + .iter() + .map(|i| (i.location.path.as_str(), i.target.as_str())) + .collect::>(), + [ + ("main.arcadia", "dir/left.arcadia"), + ("main.arcadia", "dir/right.arcadia"), + ("dir/left.arcadia", "shared.arcadia"), + ("dir/right.arcadia", "shared.arcadia"), + ] + ); + let cycle = "// 雪\r\ninclude \"main.arcadia\";"; + bundle.files.insert("shared.arcadia".into(), cycle.into()); + let before = bundle.clone(); + let errors = super::super::compile_once(&bundle, ProjectLimits::default()).unwrap_err(); + assert_eq!(errors[0].diagnostic.code, "include_cycle"); + assert_eq!(errors[0].path, "shared.arcadia"); + assert_eq!(errors[0].diagnostic.line, 2); + assert_eq!( + &cycle[errors[0].diagnostic.span.start..errors[0].diagnostic.span.end], + "include \"main.arcadia\";" + ); + assert_eq!( + errors[0].diagnostic.message, + "Include cycle: main.arcadia -> dir/left.arcadia -> shared.arcadia -> main.arcadia." + ); + assert_eq!(bundle, before); + } +}