From 02b9d5ce4c73d0446146b13b46a228e6fd4d7735 Mon Sep 17 00:00:00 2001 From: "sorokin.oleg.edu" Date: Sat, 12 Sep 2026 20:37:12 -0700 Subject: [PATCH] Stage bounded source8 expression syntax and raw token spans --- crates/oa-core/src/language.rs | 4 + .../oa-core/src/language/analysis_source.rs | 103 ++++ .../src/language/analysis_source/budget.rs | 137 +++++ .../src/language/analysis_source/lex.rs | 138 +++++ .../src/language/analysis_source/syntax.rs | 550 ++++++++++++++++++ .../src/language/analysis_source/tests.rs | 413 +++++++++++++ 6 files changed, 1345 insertions(+) create mode 100644 crates/oa-core/src/language/analysis_source.rs create mode 100644 crates/oa-core/src/language/analysis_source/budget.rs create mode 100644 crates/oa-core/src/language/analysis_source/lex.rs create mode 100644 crates/oa-core/src/language/analysis_source/syntax.rs create mode 100644 crates/oa-core/src/language/analysis_source/tests.rs diff --git a/crates/oa-core/src/language.rs b/crates/oa-core/src/language.rs index 46e7bf63..1342c092 100644 --- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -1,6 +1,10 @@ //! Typed OA/SA language, tolerant source analysis, and explicit V1/V2/V3/V4 migration. #[path = "language/semantic_source.rs"] mod semantic_source; +// Staged syntax foundation only. Public source8 admission waits for complete +// declaration lowering, source updates and the root-owned resource gate. +#[cfg(test)] +mod analysis_source; #[path = "migration_v2/source.rs"] mod v2_source; use crate::*; diff --git a/crates/oa-core/src/language/analysis_source.rs b/crates/oa-core/src/language/analysis_source.rs new file mode 100644 index 00000000..1a857522 --- /dev/null +++ b/crates/oa-core/src/language/analysis_source.rs @@ -0,0 +1,103 @@ +//! Unresolved typed source8 syntax, staged for the complete source adapter. +//! This module does not admit a Project, resolve references or enable source8. +mod budget; +mod lex; +mod syntax; + +use super::Span; +use crate::analysis::{BinaryOp, UnaryOp}; +use crate::analysis_numeric::Exact; +use budget::Budget; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error { + pub span: Span, + pub message: String, +} +fn error(span: Span, message: impl Into) -> Error { + Error { + span, + message: message.into(), + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Reference { + pub name: String, + pub span: Span, +} +#[derive(Debug, PartialEq, Eq)] +pub struct Argument { + pub parameter: Reference, + pub value: Expression, +} +#[derive(Debug, PartialEq, Eq)] +pub struct Expression { + pub kind: Kind, + pub span: Span, + pub depth: usize, + pub grouped: bool, + /// Original numeric tokens, including constructor punctuation, excluding + /// trivia. The full span retains intervening comments and spelling. + pub literal: Option, +} +#[derive(Debug, PartialEq, Eq)] +pub struct LiteralSyntax { + pub span: Span, + pub tokens: Vec, +} +#[derive(Debug, PartialEq, Eq)] +pub enum Kind { + Boolean(bool), + Number(Exact), + Quantity { + magnitude: Exact, + unit: Reference, + }, + Property(Reference), + Parameter(Reference), + List(Vec), + Unary { + op: UnaryOp, + operand: Box, + }, + Binary { + op: BinaryOp, + left: Box, + right: Box, + }, + If { + condition: Box, + when_true: Box, + when_false: Box, + }, + Call { + definition: Reference, + arguments: Vec, + }, + Sum { + values: Box, + initial: Box, + }, + Power { + base: Box, + exponent: i8, + }, + AsKind { + kind: Reference, + value: Box, + }, +} +#[derive(Debug, PartialEq, Eq)] +pub enum Binding { + Unbound(Span), + Expression(Expression), +} + +pub fn parse(source: &str, budget: &mut Budget) -> Result { + let tokens = lex::tokens(source, budget)?; + syntax::parse(source, &tokens, budget) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oa-core/src/language/analysis_source/budget.rs b/crates/oa-core/src/language/analysis_source/budget.rs new file mode 100644 index 00000000..383d7ebf --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/budget.rs @@ -0,0 +1,137 @@ +//! Candidate limits for measured adoption; one owner spans all parsed files. +use super::{Error, Span, error}; +use crate::analysis_numeric::{Exact, WorkLimits, WorkMeter}; +use crate::language::ParseLimits; + +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub bytes: usize, + pub tokens: usize, + pub entries: usize, + pub references: usize, + pub frames: usize, + pub numeric_units: u64, +} +impl Limits { + pub fn source8(caller: ParseLimits) -> Self { + let bytes = caller.max_source_bytes.min(64 * 1024 * 1024); + Self { + bytes, + tokens: bytes.min(1_048_576), + entries: caller.max_records.min(262_144), + references: caller.max_step_references.min(500_000), + frames: 256, + numeric_units: 50_000_000, + } + } +} +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Usage { + pub bytes: usize, + pub tokens: usize, + pub entries: usize, + pub references: usize, + pub deepest_frame: usize, +} +pub struct Budget { + limits: Limits, + pub usage: Usage, + numeric: WorkMeter, + failed: Option, +} +impl Budget { + pub fn new(limits: Limits) -> Result { + let hard = Limits::source8(ParseLimits::UNLIMITED); + if limits.bytes > hard.bytes + || limits.tokens > hard.tokens + || limits.entries > hard.entries + || limits.references > hard.references + || limits.frames > hard.frames + || limits.numeric_units > hard.numeric_units + { + return Err(error( + Span::default(), + "Source8 profile cannot exceed candidate hard limits.", + )); + } + Ok(Self { + limits, + usage: Usage::default(), + numeric: WorkMeter::new( + WorkLimits::new(0, limits.numeric_units) + .map_err(|e| error(Span::default(), e.to_string()))?, + ), + failed: None, + }) + } + pub fn active(&self) -> Result<(), Error> { + self.failed.clone().map_or(Ok(()), Err) + } + fn charge( + &mut self, + used: usize, + amount: usize, + maximum: usize, + span: Span, + name: &str, + ) -> Result<(), Error> { + self.active()?; + if amount > maximum.saturating_sub(used) { + let failure = error(span, format!("Source8 {name} limit exceeded.")); + self.failed = Some(failure.clone()); + return Err(failure); + } + Ok(()) + } + pub fn bytes(&mut self, amount: usize, span: Span) -> Result<(), Error> { + self.charge(self.usage.bytes, amount, self.limits.bytes, span, "bytes")?; + self.usage.bytes += amount; + Ok(()) + } + pub fn token(&mut self, span: Span) -> Result<(), Error> { + self.charge(self.usage.tokens, 1, self.limits.tokens, span, "tokens")?; + self.usage.tokens += 1; + Ok(()) + } + pub fn entry(&mut self, span: Span) -> Result<(), Error> { + self.charge( + self.usage.entries, + 1, + self.limits.entries, + span, + "syntax entries", + )?; + self.usage.entries += 1; + Ok(()) + } + pub fn reference(&mut self, span: Span) -> Result<(), Error> { + self.charge( + self.usage.references, + 1, + self.limits.references, + span, + "references", + )?; + self.usage.references += 1; + Ok(()) + } + pub fn frame(&mut self, depth: usize, span: Span) -> Result<(), Error> { + self.charge(0, depth, self.limits.frames, span, "syntax frames")?; + self.usage.deepest_frame = self.usage.deepest_frame.max(depth); + Ok(()) + } + pub fn exact(&mut self, spelling: &str, span: Span) -> Result { + self.active()?; + match Exact::parse_literal(spelling, &mut self.numeric) { + Ok(value) => Ok(value), + Err(cause) => { + let failure = error(span, cause.to_string()); + self.failed = Some(failure.clone()); + Err(failure) + } + } + } + pub fn numeric_units(&self) -> u64 { + self.numeric.usage().numeric_units + } +} diff --git a/crates/oa-core/src/language/analysis_source/lex.rs b/crates/oa-core/src/language/analysis_source/lex.rs new file mode 100644 index 00000000..0629d7cb --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/lex.rs @@ -0,0 +1,138 @@ +//! Raw numeric tokens: finite coordinate checks remain in the existing reader. +use super::{Budget, Error, Span, error}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Class { + Word, + Number, + String, + Punctuation, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Token { + pub class: Class, + pub span: Span, +} +pub fn tokens(source: &str, budget: &mut Budget) -> Result, Error> { + budget.bytes( + source.len(), + Span { + start: 0, + end: source.len(), + }, + )?; + let bytes = source.as_bytes(); + let mut cursor = 0; + let mut result = Vec::new(); + while cursor < bytes.len() { + let start = cursor; + // Reserve before scanning/storing a token, including skipped trivia. + budget.token(Span { start, end: start })?; + let class = if bytes[cursor].is_ascii_whitespace() { + while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { + cursor += 1; + } + None + } else if bytes[cursor..].starts_with(b"//") { + while cursor < bytes.len() && bytes[cursor] != b'\n' { + cursor += 1; + } + None + } else if bytes[cursor].is_ascii_alphabetic() || bytes[cursor] == b'_' { + cursor += 1; + while cursor < bytes.len() + && (bytes[cursor].is_ascii_alphanumeric() || bytes[cursor] == b'_') + { + cursor += 1; + } + Some(Class::Word) + } else if bytes[cursor].is_ascii_digit() { + while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { + cursor += 1; + } + if bytes.get(cursor) == Some(&b'.') { + cursor += 1; + let fraction = cursor; + while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { + cursor += 1; + } + if cursor == fraction { + return Err(error( + Span { start, end: cursor }, + "Expected decimal fraction digits.", + )); + } + } + if matches!(bytes.get(cursor), Some(b'e' | b'E')) { + cursor += 1; + if matches!(bytes.get(cursor), Some(b'+' | b'-')) { + cursor += 1; + } + let exponent = cursor; + while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { + cursor += 1; + } + if cursor == exponent { + return Err(error( + Span { start, end: cursor }, + "Expected exponent digits.", + )); + } + } + if bytes + .get(cursor) + .is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.')) + { + return Err(error( + Span { start, end: cursor }, + "Malformed numeric atom.", + )); + } + Some(Class::Number) + } else if bytes[cursor] == b'"' { + cursor += 1; + let mut closed = false; + while cursor < bytes.len() { + match bytes[cursor] { + b'\\' => { + cursor += 1; + if cursor < bytes.len() { + cursor += 1; + } + } + b'"' => { + cursor += 1; + closed = true; + break; + } + _ => cursor += 1, + } + } + if !closed { + return Err(error(Span { start, end: cursor }, "Unclosed string.")); + } + // Quoted field decoding remains the typed declaration consumer's job. + Some(Class::String) + } else if b"{}();,[]:=@$+-*/<>!".contains(&bytes[cursor]) { + cursor += 1; + if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') && bytes.get(cursor) == Some(&b'=') + { + cursor += 1; + } + Some(Class::Punctuation) + } else { + let end = start + source[start..].chars().next().unwrap().len_utf8(); + return Err(error( + Span { start, end }, + "Unexpected expression character.", + )); + }; + if let Some(class) = class { + result.push(Token { + class, + span: Span { start, end: cursor }, + }); + } + } + Ok(result) +} diff --git a/crates/oa-core/src/language/analysis_source/syntax.rs b/crates/oa-core/src/language/analysis_source/syntax.rs new file mode 100644 index 00000000..981bc061 --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/syntax.rs @@ -0,0 +1,550 @@ +//! Iterative Pratt machine. Source nesting does not recurse on the native stack. +use super::*; +use lex::{Class, Token}; +use std::collections::BTreeSet; + +enum SequenceKind { + List, + If, + Sum, + Unary(UnaryOp), + Call(Reference), +} +struct Sequence { + kind: SequenceKind, + start: usize, + values: Vec<(Option, Expression)>, + keys: BTreeSet, +} +enum Task { + Expression(u8), + Continue(u8), + Binary(Expression, BinaryOp), + Unary(usize, UnaryOp), + Group(usize), + Next(Sequence), + Collected(Sequence, Option), + Power(usize), + AsKind(usize, Reference), +} +struct Parser<'a, 'b> { + source: &'a str, + tokens: &'a [Token], + cursor: usize, + frames: usize, + budget: &'b mut Budget, +} +pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Result { + let mut p = Parser { + source, + tokens, + cursor: 0, + frames: 0, + budget, + }; + if p.text() == "unbound" { + let span = p.span(); + p.cursor += 1; + p.end()?; + return Ok(Binding::Unbound(span)); + } + let mut tasks = vec![Task::Expression(0)]; + let mut values = Vec::new(); + while let Some(task) = tasks.pop() { + p.budget.active()?; + match task { + Task::Expression(minimum) => { + let start = p.span().start; + if p.text() != "(" { + p.budget.entry(p.span())?; + } + tasks.push(Task::Continue(minimum)); + match p.text() { + "true" | "false" => { + let value = p.text() == "true"; + let span = p.span(); + p.cursor += 1; + values.push(p.node(Kind::Boolean(value), span)?); + } + "-" => { + p.open()?; + p.cursor += 1; + tasks.push(Task::Unary(start, UnaryOp::Negate)); + tasks.push(Task::Expression(7)); + } + "(" => { + p.open()?; + p.cursor += 1; + tasks.push(Task::Group(start)); + tasks.push(Task::Expression(0)); + } + "[" => { + p.open()?; + p.cursor += 1; + tasks.push(Task::Next(Sequence::new(SequenceKind::List, start))); + } + "@" => { + p.cursor += 1; + let reference = p.reference()?; + if p.text() == "(" { + p.open()?; + p.cursor += 1; + tasks.push(Task::Next(Sequence::new( + SequenceKind::Call(reference), + start, + ))); + } else { + let end = reference.span.end; + values.push(p.node(Kind::Property(reference), Span { start, end })?); + } + } + "$" => { + p.cursor += 1; + let reference = p.reference()?; + let end = reference.span.end; + values.push(p.node(Kind::Parameter(reference), Span { start, end })?); + } + "ratio" => { + let (value, literal) = p.ratio()?; + values.push(p.number(value, literal)?); + } + "abs" | "not" | "if" | "sum" => { + let kind = match p.text() { + "abs" => SequenceKind::Unary(UnaryOp::Absolute), + "not" => SequenceKind::Unary(UnaryOp::Not), + "if" => SequenceKind::If, + _ => SequenceKind::Sum, + }; + p.cursor += 1; + p.open()?; + p.take("(")?; + tasks.push(Task::Next(Sequence::new(kind, start))); + } + "power" => { + p.cursor += 1; + p.open()?; + p.take("(")?; + tasks.push(Task::Power(start)); + tasks.push(Task::Expression(0)); + } + "as_kind" => { + p.cursor += 1; + p.open()?; + p.take("(")?; + p.take("@")?; + let kind = p.reference()?; + p.take(",")?; + tasks.push(Task::AsKind(start, kind)); + tasks.push(Task::Expression(0)); + } + _ if p + .tokens + .get(p.cursor) + .is_some_and(|token| token.class == Class::Number) => + { + let span = p.span(); + let value = p.budget.exact(&p.source[span.start..span.end], span)?; + p.cursor += 1; + values.push(p.number( + value, + LiteralSyntax { + span, + tokens: vec![span], + }, + )?); + } + _ => return Err(error(p.span(), "Expected an analysis expression.")), + } + } + Task::Continue(minimum) => { + if let Some((op, precedence)) = + operator(p.text()).filter(|(_, prec)| *prec >= minimum) + { + let left = values.pop().expect("completed Pratt operand"); + if comparison(op) && ungrouped_comparison(&left) { + return Err(error(p.span(), "Comparison children require parentheses.")); + } + p.budget.entry(p.span())?; + p.cursor += 1; + tasks.push(Task::Continue(minimum)); + tasks.push(Task::Binary(left, op)); + tasks.push(Task::Expression(precedence + 1)); + } + } + Task::Binary(left, op) => { + let right = values.pop().expect("completed right operand"); + if comparison(op) && ungrouped_comparison(&right) { + return Err(error( + right.span, + "Comparison children require parentheses.", + )); + } + let span = Span { + start: left.span.start, + end: right.span.end, + }; + values.push(p.node( + Kind::Binary { + op, + left: Box::new(left), + right: Box::new(right), + }, + span, + )?); + } + Task::Unary(start, op) => { + let operand = values.pop().expect("completed unary operand"); + let end = operand.span.end; + p.close(); + values.push(p.node( + Kind::Unary { + op, + operand: Box::new(operand), + }, + Span { start, end }, + )?); + } + Task::Group(start) => { + let end = p.take(")")?.end; + p.close(); + let value = values.last_mut().expect("completed grouped expression"); + value.grouped = true; + value.span = Span { start, end }; + } + Task::Next(mut sequence) => { + let closing = if matches!(sequence.kind, SequenceKind::List) { + "]" + } else { + ")" + }; + if p.text() == closing { + let end = p.take(closing)?.end; + p.close(); + values.push(p.sequence(sequence, end)?); + } else { + let count = sequence.values.len(); + let maximum = match sequence.kind { + SequenceKind::List => crate::analysis::MAX_LIST_LENGTH, + SequenceKind::Unary(_) => 1, + SequenceKind::If => 3, + SequenceKind::Sum => 2, + SequenceKind::Call(_) => usize::MAX, + }; + if count >= maximum { + return Err(error( + p.span(), + "Too many list members or constructor arguments.", + )); + } + let key = if matches!(sequence.kind, SequenceKind::Call(_)) { + p.budget.entry(p.span())?; + let key = p.reference()?; + if !sequence.keys.insert(key.name.clone()) { + return Err(error(key.span, "Duplicate call argument name.")); + } + p.take(":")?; + Some(key) + } else { + None + }; + tasks.push(Task::Collected(sequence, key)); + tasks.push(Task::Expression(0)); + } + } + Task::Collected(mut sequence, key) => { + sequence + .values + .push((key, values.pop().expect("completed sequence value"))); + let closing = if matches!(sequence.kind, SequenceKind::List) { + "]" + } else { + ")" + }; + if p.text() != closing { + p.take(",")?; + if p.text() == closing { + return Err(error(p.span(), "Trailing comma is not an expression.")); + } + } + tasks.push(Task::Next(sequence)); + } + Task::Power(start) => { + let base = values.pop().expect("completed power base"); + p.take(",")?; + let negative = if p.text() == "-" { + p.cursor += 1; + true + } else { + false + }; + let span = p.integer()?; + let magnitude = p.source[span.start..span.end] + .parse::() + .map_err(|_| error(span, "Power exponent is outside -16..16."))?; + if magnitude > 16 { + return Err(error(span, "Power exponent is outside -16..16.")); + } + let exponent = if negative { + -(magnitude as i8) + } else { + magnitude as i8 + }; + let end = p.take(")")?.end; + p.close(); + values.push(p.node( + Kind::Power { + base: Box::new(base), + exponent, + }, + Span { start, end }, + )?); + } + Task::AsKind(start, kind) => { + let value = values.pop().expect("completed kind assertion value"); + let end = p.take(")")?.end; + p.close(); + values.push(p.node( + Kind::AsKind { + kind, + value: Box::new(value), + }, + Span { start, end }, + )?); + } + } + } + p.end()?; + Ok(Binding::Expression( + values.pop().expect("one complete expression"), + )) +} +impl Sequence { + fn new(kind: SequenceKind, start: usize) -> Self { + Self { + kind, + start, + values: Vec::new(), + keys: BTreeSet::new(), + } + } +} +impl Parser<'_, '_> { + fn span(&self) -> Span { + self.tokens.get(self.cursor).map_or( + Span { + start: self.source.len(), + end: self.source.len(), + }, + |token| token.span, + ) + } + fn text(&self) -> &str { + let span = self.span(); + &self.source[span.start..span.end] + } + fn take(&mut self, expected: &str) -> Result { + if self.text() != expected { + return Err(error(self.span(), format!("Expected '{expected}'."))); + } + let span = self.span(); + self.cursor += 1; + Ok(span) + } + fn end(&self) -> Result<(), Error> { + if self.cursor != self.tokens.len() { + Err(error( + self.span(), + "Unexpected content after analysis expression.", + )) + } else { + Ok(()) + } + } + fn open(&mut self) -> Result<(), Error> { + self.budget.frame(self.frames + 1, self.span())?; + self.frames += 1; + Ok(()) + } + fn close(&mut self) { + self.frames -= 1; + } + fn reference(&mut self) -> Result { + let span = self.span(); + if !self + .tokens + .get(self.cursor) + .is_some_and(|token| token.class == Class::Word) + || !crate::valid_alias(self.text()) + { + return Err(error(span, "Expected a valid reference or parameter name.")); + } + self.budget.reference(span)?; + let name = self.text().to_owned(); + self.cursor += 1; + Ok(Reference { name, span }) + } + fn integer(&mut self) -> Result { + let span = self.span(); + let text = self.text(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return Err(error(span, "Expected unsigned integer digits.")); + } + if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS { + return Err(error(span, "Integer component exceeds digit limit.")); + } + self.cursor += 1; + Ok(span) + } + fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> { + let first_token = self.cursor; + let start = self.take("ratio")?.start; + self.open()?; + self.take("(")?; + let negative = if self.text() == "-" { + self.cursor += 1; + true + } else { + false + }; + let numerator = self.integer()?; + self.take(",")?; + let denominator = self.integer()?; + let end = self.take(")")?.end; + self.close(); + let n = &self.source[numerator.start..numerator.end]; + let d = &self.source[denominator.start..denominator.end]; + let length = 9 + usize::from(negative) + n.len() + d.len(); + if length > crate::analysis_numeric::MAX_LITERAL_BYTES { + return Err(error( + Span { start, end }, + "Numeric reconstruction exceeds literal byte limit.", + )); + } + let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" }); + Ok(( + self.budget.exact(&normalized, Span { start, end })?, + LiteralSyntax { + span: Span { start, end }, + tokens: self.tokens[first_token..self.cursor] + .iter() + .map(|token| token.span) + .collect(), + }, + )) + } + fn number(&mut self, magnitude: Exact, literal: LiteralSyntax) -> Result { + let Span { start, end } = literal.span; + let mut expression = if self.text() == "[" { + self.open()?; + self.cursor += 1; + self.take("@")?; + let unit = self.reference()?; + let end = self.take("]")?.end; + self.close(); + self.node(Kind::Quantity { magnitude, unit }, Span { start, end }) + } else { + self.node(Kind::Number(magnitude), Span { start, end }) + }?; + expression.literal = Some(literal); + Ok(expression) + } + fn node(&mut self, kind: Kind, span: Span) -> Result { + let child_depth = match &kind { + Kind::Unary { operand, .. } => operand.depth, + Kind::Binary { left, right, .. } => left.depth.max(right.depth), + Kind::If { + condition, + when_true, + when_false, + } => condition.depth.max(when_true.depth).max(when_false.depth), + Kind::List(values) => values.iter().map(|v| v.depth).max().unwrap_or(0), + Kind::Call { arguments, .. } => { + arguments.iter().map(|a| a.value.depth).max().unwrap_or(0) + } + Kind::Sum { values, initial } => values.depth.max(initial.depth), + Kind::Power { base, .. } => base.depth, + Kind::AsKind { value, .. } => value.depth, + _ => 0, + }; + if child_depth >= crate::analysis::MAX_EXPRESSION_DEPTH { + return Err(error(span, "Expression logical depth exceeds 128.")); + } + Ok(Expression { + kind, + span, + depth: child_depth + 1, + grouped: false, + literal: None, + }) + } + fn sequence(&mut self, sequence: Sequence, end: usize) -> Result { + let span = Span { + start: sequence.start, + end, + }; + let mut values = sequence.values.into_iter(); + let mut next = || { + values + .next() + .map(|(_, value)| Box::new(value)) + .ok_or_else(|| error(span, "Missing required constructor argument.")) + }; + let kind = match sequence.kind { + SequenceKind::If => Kind::If { + condition: next()?, + when_true: next()?, + when_false: next()?, + }, + SequenceKind::Sum => Kind::Sum { + values: next()?, + initial: next()?, + }, + SequenceKind::Unary(op) => Kind::Unary { + op, + operand: next()?, + }, + SequenceKind::List => Kind::List(values.map(|(_, value)| value).collect()), + SequenceKind::Call(definition) => Kind::Call { + definition, + arguments: values + .map(|(parameter, value)| Argument { + parameter: parameter.expect("parsed named argument"), + value, + }) + .collect(), + }, + }; + self.node(kind, span) + } +} +fn comparison(op: BinaryOp) -> bool { + matches!( + op, + BinaryOp::Equal + | BinaryOp::NotEqual + | BinaryOp::Less + | BinaryOp::LessOrEqual + | BinaryOp::Greater + | BinaryOp::GreaterOrEqual + ) +} +fn ungrouped_comparison(value: &Expression) -> bool { + !value.grouped && matches!(&value.kind, Kind::Binary { op, .. } if comparison(*op)) +} +fn operator(text: &str) -> Option<(BinaryOp, u8)> { + Some(match text { + "or" => (BinaryOp::Or, 1), + "and" => (BinaryOp::And, 2), + "==" => (BinaryOp::Equal, 3), + "!=" => (BinaryOp::NotEqual, 3), + "<" => (BinaryOp::Less, 4), + "<=" => (BinaryOp::LessOrEqual, 4), + ">" => (BinaryOp::Greater, 4), + ">=" => (BinaryOp::GreaterOrEqual, 4), + "+" => (BinaryOp::Add, 5), + "-" => (BinaryOp::Subtract, 5), + "*" => (BinaryOp::Multiply, 6), + "/" => (BinaryOp::Divide, 6), + _ => return None, + }) +} diff --git a/crates/oa-core/src/language/analysis_source/tests.rs b/crates/oa-core/src/language/analysis_source/tests.rs new file mode 100644 index 00000000..83ae317f --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/tests.rs @@ -0,0 +1,413 @@ +use super::*; +use crate::language::ParseLimits; +use budget::{Budget, Limits}; + +fn limits() -> Limits { + Limits::source8(ParseLimits::UNLIMITED) +} +fn budget() -> Budget { + Budget::new(limits()).unwrap() +} +fn expression(source: &str) -> Expression { + match parse(source, &mut budget()).unwrap() { + Binding::Expression(value) => value, + Binding::Unbound(_) => panic!("expected expression"), + } +} +fn number(value: &Expression) -> String { + let Kind::Number(exact) = &value.kind else { + panic!("expected Number"); + }; + exact.canonical_literal() +} + +#[test] +fn significant_ratio_and_quantity_trivia_preserve_original_spans() { + let source = "ratio // explanation café\n ( - // sign\n 3, // denominator\n 4 ) // measurement\n [@millisecond]"; + let parsed = expression(source); + assert_eq!( + parsed.span, + Span { + start: 0, + end: source.len() + } + ); + let literal = parsed.literal.as_ref().unwrap(); + assert!(source[literal.span.start..literal.span.end].contains("// denominator")); + assert_eq!( + literal + .tokens + .iter() + .map(|span| &source[span.start..span.end]) + .collect::>(), + ["ratio", "(", "-", "3", ",", "4", ")"] + ); + let Kind::Quantity { magnitude, unit } = parsed.kind else { + panic!("expected Quantity"); + }; + assert_eq!(magnitude.canonical_literal(), "ratio(-3, 4)"); + assert_eq!(unit.name, "millisecond"); + assert_eq!(&source[unit.span.start..unit.span.end], "millisecond"); + let mut renamed = source.to_owned(); + renamed.replace_range(unit.span.start..unit.span.end, "ms"); + assert_eq!(renamed, source.replacen("millisecond", "ms", 1)); + assert_eq!( + number(&expression("ratio (1, // rationale\n 2)")), + "ratio(1, 2)" + ); + for invalid in [ + "1 e - 2", + "1e // x\n -2", + "1 . 2", + "ratio(1 + 2, 3)", + "ratio(1, -2)", + "ratio(+1, 2)", + "ratio(1.0, 2)", + "ratio(1, 0)", + ] { + assert!(parse(invalid, &mut budget()).is_err(), "{invalid}"); + } +} + +#[test] +fn raw_numeric_lexing_preserves_exact_atoms_and_operator_boundaries() { + let source = "1-2 1e-2 ratio(-1,2) -(1) <= >= == !="; + let tokens = lex::tokens(source, &mut budget()).unwrap(); + let text: Vec<_> = tokens + .iter() + .map(|t| &source[t.span.start..t.span.end]) + .collect(); + assert_eq!( + text, + [ + "1", "-", "2", "1e-2", "ratio", "(", "-", "1", ",", "2", ")", "-", "(", "1", ")", "<=", + ">=", "==", "!=" + ] + ); + let huge = "9".repeat(400); + assert_eq!( + lex::tokens(&huge, &mut budget()).unwrap()[0].class, + lex::Class::Number + ); + assert_eq!(number(&expression(&huge)), huge); + for invalid in ["1e", "1e+", "1_0", "0x1", "1.2.3"] { + assert!(lex::tokens(invalid, &mut budget()).is_err(), "{invalid}"); + } + let quoted = "\"quoted\\\" café\""; + let tokens = lex::tokens(quoted, &mut budget()).unwrap(); + assert_eq!(tokens[0].class, lex::Class::String); + assert_eq!("ed[tokens[0].span.start..tokens[0].span.end], quoted); +} + +#[test] +fn unary_literals_precedence_and_comparison_children_keep_distinct_trees() { + assert_eq!(number(&expression("ratio(-1, 1)")), "ratio(-1, 1)"); + let negative = expression("-(1)"); + assert!(matches!( + negative.kind, + Kind::Unary { + op: UnaryOp::Negate, + .. + } + )); + let left = expression("(1 - 2) - 3"); + let right = expression("1 - (2 - 3)"); + let Kind::Binary { + left, right: last, .. + } = left.kind + else { + panic!(); + }; + assert!(matches!( + left.kind, + Kind::Binary { + op: BinaryOp::Subtract, + .. + } + )); + assert_eq!(number(&last), "3"); + let Kind::Binary { + left: first, right, .. + } = right.kind + else { + panic!(); + }; + assert_eq!(number(&first), "1"); + assert!(matches!( + right.kind, + Kind::Binary { + op: BinaryOp::Subtract, + .. + } + )); + for valid in [ + "(1 < 2) == true", + "(true == false) == true", + "true == (1 < 2)", + "1 + 2 * 3", + "not(false) and true or false", + ] { + parse(valid, &mut budget()).unwrap(); + } + for invalid in [ + "1 < 2 == true", + "true == false == true", + "1 < 2 < 3", + "true == 1 < 2", + ] { + assert!(parse(invalid, &mut budget()).is_err(), "{invalid}"); + } +} + +#[test] +fn every_expression_shape_and_exact_constructor_arity_is_represented() { + let tree = expression("if(true, sum([@p, $x], 0), as_kind(@duration, power(abs(-(2)), -2)))"); + let Kind::If { + condition, + when_true, + when_false, + } = tree.kind + else { + panic!(); + }; + assert_eq!(condition.kind, Kind::Boolean(true)); + let Kind::Sum { values, initial } = when_true.kind else { + panic!(); + }; + assert_eq!(number(&initial), "0"); + let Kind::List(values) = values.kind else { + panic!(); + }; + assert!(matches!(&values[0].kind, Kind::Property(r) if r.name == "p")); + assert!(matches!(&values[1].kind, Kind::Parameter(r) if r.name == "x")); + let Kind::AsKind { kind, value } = when_false.kind else { + panic!(); + }; + assert_eq!(kind.name, "duration"); + assert!(matches!(value.kind, Kind::Power { exponent: -2, .. })); + assert!(matches!(expression("[]").kind, Kind::List(v) if v.is_empty())); + assert!( + matches!(expression("@zero()").kind, Kind::Call { arguments, .. } if arguments.is_empty()) + ); + assert!(matches!( + parse(" unbound // pending", &mut budget()).unwrap(), + Binding::Unbound(_) + )); + for invalid in [ + "sum([1])", + "sum([1], 0, 2)", + "if(true, 1)", + "abs()", + "not(true, false)", + "power(2, 17)", + "power(2, 1.0)", + "as_kind(duration, 1)", + "[1,]", + "[1 2]", + "(1+2)[@unit]", + ] { + assert!(parse(invalid, &mut budget()).is_err(), "{invalid}"); + } +} + +#[test] +fn repeated_call_argument_occurrences_have_distinct_original_spans() { + let source = "@combine(a: @p, b: 2) + @combine(a: @q, b: 4)"; + let tree = expression(source); + let Kind::Binary { left, right, .. } = tree.kind else { + panic!(); + }; + let Kind::Call { + definition: first, + arguments: a, + } = left.kind + else { + panic!(); + }; + let Kind::Call { + definition: second, + arguments: b, + } = right.kind + else { + panic!(); + }; + assert_eq!(first.name, second.name); + assert_ne!(first.span, second.span); + assert_eq!(a[0].parameter.name, b[0].parameter.name); + assert_ne!(a[0].parameter.span, b[0].parameter.span); + for argument in a.iter().chain(&b) { + assert_eq!( + &source[argument.parameter.span.start..argument.parameter.span.end], + argument.parameter.name + ); + } + assert!( + parse("@combine(a: 1, a: 2)", &mut budget()) + .unwrap_err() + .message + .contains("Duplicate") + ); + assert!(parse("@combine(a: 1, b:)", &mut budget()).is_err()); + // Resolution to owner-qualified ParameterId and actual source rewrites are + // deliberately a later stage; this oracle establishes unique syntax spans. +} + +#[test] +fn budgets_aggregate_across_fragments_and_latch_numeric_exhaustion() { + let mut first = budget(); + parse("ratio(1, 3)", &mut first).unwrap(); + let cost = first.numeric_units(); + assert!(cost > 0); + let mut settings = limits(); + settings.numeric_units = cost; + let mut shared = Budget::new(settings).unwrap(); + parse("ratio (1, // same numeric cost\n 3)", &mut shared).unwrap(); + assert_eq!(shared.numeric_units(), cost); + let failure = parse("2", &mut shared).unwrap_err(); + let usage = shared.usage; + assert_eq!(parse("unbound", &mut shared).unwrap_err(), failure); + assert_eq!(shared.usage, usage); + parse("2", &mut budget()).unwrap(); + + let mut settings = limits(); + settings.entries = 2; + let mut shared = Budget::new(settings).unwrap(); + parse("true", &mut shared).unwrap(); + parse("false", &mut shared).unwrap(); + assert!( + parse("true", &mut shared) + .unwrap_err() + .message + .contains("entries") + ); + assert_eq!(shared.usage.entries, 2); + let mut settings = limits(); + settings.references = 1; + let mut shared = Budget::new(settings).unwrap(); + parse("@one", &mut shared).unwrap(); + assert!( + parse("$two", &mut shared) + .unwrap_err() + .message + .contains("references") + ); + assert_eq!(shared.usage.references, 1); +} + +#[test] +fn byte_token_frame_logical_and_list_boundaries_have_nearby_successes() { + let mut settings = limits(); + settings.bytes = 4; + let mut shared = Budget::new(settings).unwrap(); + parse("true", &mut shared).unwrap(); + assert!( + parse(" ", &mut shared) + .unwrap_err() + .message + .contains("bytes") + ); + assert_eq!(shared.usage.bytes, 4); + let mut settings = limits(); + settings.tokens = 2; + let mut shared = Budget::new(settings).unwrap(); + parse("true ", &mut shared).unwrap(); + assert!( + parse("// note", &mut shared) + .unwrap_err() + .message + .contains("tokens") + ); + assert_eq!(shared.usage.tokens, 2); + let nested = |depth: usize| format!("{}true{}", "(".repeat(depth), ")".repeat(depth)); + let mut shared = budget(); + assert_eq!( + match parse(&nested(256), &mut shared).unwrap() { + Binding::Expression(e) => e.depth, + _ => 0, + }, + 1 + ); + assert_eq!(shared.usage.deepest_frame, 256); + assert!( + parse(&nested(257), &mut budget()) + .unwrap_err() + .message + .contains("frames") + ); + let unary = |depth: usize| format!("{}true{}", "not(".repeat(depth - 1), ")".repeat(depth - 1)); + assert_eq!(expression(&unary(128)).depth, 128); + assert!( + parse(&unary(129), &mut budget()) + .unwrap_err() + .message + .contains("logical depth") + ); + let list = |count: usize| format!("[{}]", vec!["true"; count].join(",")); + parse(&list(1024), &mut budget()).unwrap(); + assert!(parse(&list(1025), &mut budget()).is_err()); + let lowered = Limits::source8(ParseLimits { + max_source_bytes: 8, + max_graph_objects: 3, + max_records: 5, + max_step_references: 2, + }); + assert_eq!( + ( + lowered.bytes, + lowered.tokens, + lowered.entries, + lowered.references + ), + (8, 8, 5, 2) + ); + let mut above = limits(); + above.frames += 1; + assert!(Budget::new(above).is_err()); +} + +#[test] +fn partial_wide_and_deep_input_cleanup_does_not_poison_a_fresh_parse() { + let source = "@f(a: if(true, [ratio (1, //why\n 2), @p], sum([1, 2], 0)), b: power(2, -3))"; + let tokens = lex::tokens(source, &mut budget()).unwrap(); + for token in tokens.iter().filter(|token| { + matches!( + &source[token.span.start..token.span.end], + "(" | "[" | "," | ":" | "@" + ) + }) { + assert!(parse(&source[..token.span.end], &mut budget()).is_err()); + assert_eq!(number(&expression("ratio(2, 3)")), "ratio(2, 3)"); + } + for malformed in [ + format!("{}true", "(".repeat(256)), + format!("[{}", vec!["true"; 1024].join(",")), + "-".repeat(10_000), + ] { + assert!(parse(&malformed, &mut budget()).is_err()); + parse("true", &mut budget()).unwrap(); + } +} + +#[test] +fn public_source_headers_and_existing_numeric_token_controls_are_unchanged() { + assert!(crate::language::parse("arcadia 8; project p \"P\" {}").is_err()); + for accepted in ["-1.5", "1e-2", "0", "-0"] { + let tokens = crate::language::tokenize(accepted); + assert_eq!(tokens.len(), 1); + assert_eq!(tokens[0].class, crate::language::TokenClass::Number); + assert_eq!( + tokens[0].span, + Span { + start: 0, + end: accepted.len() + } + ); + } + for rejected in ["+1", "1_0", "0x1", "1e"] { + assert!( + crate::language::tokenize(rejected) + .iter() + .any(|t| t.class == crate::language::TokenClass::Error) + ); + } +}