From 021bdc5a921885d1f5ce5b3a4b9cdbc03a73ab68 Mon Sep 17 00:00:00 2001 From: "sorokin.oleg.edu" Date: Sat, 12 Sep 2026 21:59:20 -0700 Subject: [PATCH] feat(source): stage typed declaration syntax on shared raw tokens --- crates/oa-core/src/language.rs | 82 +-- .../oa-core/src/language/analysis_source.rs | 7 +- .../src/language/analysis_source/cursor.rs | 171 +++++ .../language/analysis_source/declarations.rs | 629 ++++++++++++++++++ .../analysis_source/declarations/tests.rs | 531 +++++++++++++++ .../src/language/analysis_source/lex.rs | 136 +--- .../src/language/analysis_source/syntax.rs | 141 +--- crates/oa-core/src/language/raw_tokens.rs | 347 ++++++++++ 8 files changed, 1719 insertions(+), 325 deletions(-) create mode 100644 crates/oa-core/src/language/analysis_source/cursor.rs create mode 100644 crates/oa-core/src/language/analysis_source/declarations.rs create mode 100644 crates/oa-core/src/language/analysis_source/declarations/tests.rs create mode 100644 crates/oa-core/src/language/raw_tokens.rs diff --git a/crates/oa-core/src/language.rs b/crates/oa-core/src/language.rs index 1b48474e..2a5b6525 100644 --- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -2,6 +2,7 @@ #[path = "language/semantic_source.rs"] mod semantic_source; mod source3_bridge; +mod raw_tokens; // Staged syntax foundation only. Public source8 admission waits for complete // declaration lowering, source updates and the root-owned resource gate. #[cfg(test)] @@ -152,86 +153,7 @@ pub struct Analysis { /// Tolerant highlighting tokens cover the whole input, including incomplete /// strings and unexpected characters. No model is created or modified. pub fn tokenize(source: &str) -> Vec { - let mut tokens = Vec::new(); - let mut cursor = 0; - while cursor < source.len() { - let start = cursor; - let ch = source[cursor..].chars().next().expect("in bounds"); - let class = if matches!(ch, ' ' | '\t' | '\r' | '\n') { - cursor += 1; - while source - .as_bytes() - .get(cursor) - .is_some_and(|next| matches!(*next, b' ' | b'\t' | b'\r' | b'\n')) - { - cursor += 1; - } - TokenClass::Whitespace - } else if source[cursor..].starts_with("//") { - cursor += source[cursor..].find('\n').unwrap_or(source.len() - cursor); - TokenClass::Comment - } else if ch == '"' { - cursor += 1; - let mut escaped = false; - let mut closed = false; - while cursor < source.len() { - let next = source[cursor..].chars().next().expect("in bounds"); - cursor += next.len_utf8(); - if escaped { - escaped = false; - } else if next == '\\' { - escaped = true; - } else if next == '"' { - closed = true; - break; - } - } - if closed && serde_json::from_str::(&source[start..cursor]).is_ok() { - TokenClass::String - } else { - TokenClass::Error - } - } else if ch.is_ascii_alphabetic() || ch == '_' { - cursor += 1; - while source - .as_bytes() - .get(cursor) - .is_some_and(|next| next.is_ascii_alphanumeric() || *next == b'_') - { - cursor += 1; - } - if reserved(&source[start..cursor]) - || matches!(&source[start..cursor], "true" | "false" | "none") - { - TokenClass::Keyword - } else { - TokenClass::Identifier - } - } else if ch.is_ascii_digit() || matches!(ch, '-' | '+' | '.') { - cursor += 1; - while source.as_bytes().get(cursor).is_some_and(|next| { - next.is_ascii_alphanumeric() || matches!(*next, b'.' | b'+' | b'-' | b'_') - }) { - cursor += 1; - } - if serde_json::from_str::(&source[start..cursor]).is_ok() { - TokenClass::Number - } else { - TokenClass::Error - } - } else if "{}();,[]:=@".contains(ch) { - cursor += ch.len_utf8(); - TokenClass::Punctuation - } else { - cursor += ch.len_utf8(); - TokenClass::Error - }; - tokens.push(TokenSpan { - class, - span: Span { start, end: cursor }, - }); - } - tokens + raw_tokens::legacy(source) } pub fn lint(source: &str) -> Vec { diff --git a/crates/oa-core/src/language/analysis_source.rs b/crates/oa-core/src/language/analysis_source.rs index 425e4cb2..42e2b8d2 100644 --- a/crates/oa-core/src/language/analysis_source.rs +++ b/crates/oa-core/src/language/analysis_source.rs @@ -1,6 +1,8 @@ //! Unresolved typed source8 syntax, staged for the complete source adapter. //! This module does not admit a Project, resolve references or enable source8. mod budget; +mod cursor; +mod declarations; mod lex; mod syntax; @@ -96,7 +98,10 @@ pub enum Binding { pub fn parse(source: &str, budget: &mut Budget) -> Result { let tokens = lex::tokens(source, budget)?; - syntax::parse(source, &tokens, budget) + let mut cursor = cursor::Cursor::new(source, &tokens, budget); + let binding = syntax::binding(&mut cursor)?; + cursor.end()?; + Ok(binding) } #[cfg(test)] diff --git a/crates/oa-core/src/language/analysis_source/cursor.rs b/crates/oa-core/src/language/analysis_source/cursor.rs new file mode 100644 index 00000000..84723258 --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/cursor.rs @@ -0,0 +1,171 @@ +//! A borrowed significant-token cursor. Spans always refer to the full source. +use super::*; +use lex::{Class, Token}; + +pub(super) struct Cursor<'a, 'b> { + pub source: &'a str, + pub tokens: &'a [Token], + pub cursor: usize, + pub frames: usize, + pub budget: &'b mut Budget, +} +impl<'a, 'b> Cursor<'a, 'b> { + pub(super) fn new(source: &'a str, tokens: &'a [Token], budget: &'b mut Budget) -> Self { + Self { + source, + tokens, + cursor: 0, + frames: 0, + budget, + } + } +} +impl Cursor<'_, '_> { + pub(super) fn span(&self) -> Span { + self.tokens.get(self.cursor).map_or( + Span { + start: self.source.len(), + end: self.source.len(), + }, + |token| token.span, + ) + } + pub(super) fn text(&self) -> &str { + let span = self.span(); + &self.source[span.start..span.end] + } + pub(super) fn take(&mut self, expected: &str) -> Result { + if self.text() != expected { + return Err(error(self.span(), format!("Expected '{expected}'."))); + } + let span = self.span(); + self.cursor += 1; + Ok(span) + } + pub(super) fn end(&self) -> Result<(), Error> { + if self.cursor != self.tokens.len() { + Err(error( + self.span(), + "Unexpected content after analysis expression.", + )) + } else { + Ok(()) + } + } + pub(super) fn open(&mut self) -> Result<(), Error> { + self.budget.frame(self.frames + 1, self.span())?; + self.frames += 1; + Ok(()) + } + pub(super) fn close(&mut self) { + self.frames -= 1; + } + pub(super) fn reference(&mut self) -> Result { + self.budget.reference(self.span())?; + self.name() + } + pub(super) fn name(&mut self) -> Result { + let span = self.span(); + if !self + .tokens + .get(self.cursor) + .is_some_and(|token| token.class == Class::Word) + || !crate::valid_alias(self.text()) + { + return Err(error(span, "Expected a valid reference or parameter name.")); + } + let name = self.text().to_owned(); + self.cursor += 1; + Ok(Reference { name, span }) + } + pub(super) fn quoted(&mut self) -> Result { + let span = self.span(); + if !self + .tokens + .get(self.cursor) + .is_some_and(|token| token.class == Class::String) + { + return Err(error(span, "Expected a quoted string.")); + } + let value = + serde_json::from_str(self.text()).map_err(|_| error(span, "Invalid quoted string."))?; + self.cursor += 1; + Ok(value) + } + pub(super) fn last_end(&self) -> usize { + self.tokens[self.cursor - 1].span.end + } + pub(super) fn exact_literal(&mut self) -> Result<(Exact, LiteralSyntax), Error> { + if self.text() == "ratio" { + return self.ratio(); + } + let span = self.span(); + if !self + .tokens + .get(self.cursor) + .is_some_and(|token| token.class == Class::Number) + { + return Err(error(span, "Expected an exact numeric atom or ratio.")); + } + let value = self + .budget + .exact(&self.source[span.start..span.end], span)?; + self.cursor += 1; + Ok(( + value, + LiteralSyntax { + span, + tokens: vec![span], + }, + )) + } + pub(super) fn integer(&mut self) -> Result { + let span = self.span(); + let text = self.text(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return Err(error(span, "Expected unsigned integer digits.")); + } + if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS { + return Err(error(span, "Integer component exceeds digit limit.")); + } + self.cursor += 1; + Ok(span) + } + pub(super) fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> { + let first_token = self.cursor; + let start = self.take("ratio")?.start; + self.open()?; + self.take("(")?; + let negative = if self.text() == "-" { + self.cursor += 1; + true + } else { + false + }; + let numerator = self.integer()?; + self.take(",")?; + let denominator = self.integer()?; + let end = self.take(")")?.end; + self.close(); + let n = &self.source[numerator.start..numerator.end]; + let d = &self.source[denominator.start..denominator.end]; + let length = 9 + usize::from(negative) + n.len() + d.len(); + if length > crate::analysis_numeric::MAX_LITERAL_BYTES { + return Err(error( + Span { start, end }, + "Numeric reconstruction exceeds literal byte limit.", + )); + } + let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" }); + Ok(( + self.budget.exact(&normalized, Span { start, end })?, + LiteralSyntax { + span: Span { start, end }, + tokens: self.tokens[first_token..self.cursor] + .iter() + .map(|token| token.span) + .collect(), + }, + )) + } +} diff --git a/crates/oa-core/src/language/analysis_source/declarations.rs b/crates/oa-core/src/language/analysis_source/declarations.rs new file mode 100644 index 00000000..89337f7e --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/declarations.rs @@ -0,0 +1,629 @@ +//! Required typed declaration syntax. No model construction or resolution. +use super::cursor::Cursor; +use super::*; +use crate::{Id, Layer}; +use std::collections::BTreeSet; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeclarationKind { + QuantityKind, + Unit, + Calculation, + Property(Layer), + Check(Layer), + Case(Layer), +} + +/// Contextual recognition only: never add these words to the global alias ban. +pub fn declaration_kind(word: &str) -> Option { + use DeclarationKind::*; + Some(match word { + "quantity_kind" => QuantityKind, + "unit" => Unit, + "calculation" => Calculation, + "oa_property" => Property(Layer::Oa), + "sa_property" => Property(Layer::Sa), + "oa_check" => Check(Layer::Oa), + "sa_check" => Check(Layer::Sa), + "oa_analysis" => Case(Layer::Oa), + "sa_analysis" => Case(Layer::Sa), + _ => return None, + }) +} + +#[derive(Debug, PartialEq, Eq)] +pub struct IdentitySyntax { + /// Missing IDs stay unresolved here. The bundle stabilization owner assigns + /// and inserts an ID once, then reparses the actual returned bytes. + pub value: Option, + pub keyword: Option, + pub literal: Option, + pub insertion: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Header { + pub kind: DeclarationKind, + pub kind_span: Span, + pub alias: Reference, + pub name: String, + pub name_span: Span, + pub identity: IdentitySyntax, + pub span: Span, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Field { + /// Complete field including its keyword and trailing semicolon. + pub span: Span, + pub keyword: Span, + /// Same meaning as an ordinary top-level field value span. + pub value_span: Span, + pub value: T, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ScalarSyntax { + pub span: Span, + pub kind: ScalarKind, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ScalarKind { + Boolean, + Number, + Quantity(Reference), +} + +#[derive(Debug, PartialEq, Eq)] +pub struct TypeSyntax { + pub span: Span, + pub scalar: ScalarSyntax, + pub list: bool, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ParameterSyntax { + pub span: Span, + pub name: Reference, + pub identity: IdentitySyntax, + pub value_type: TypeSyntax, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct DimensionEntry { + pub axis: Span, + pub exponent: Span, + pub span: Span, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct DimensionSyntax { + pub exponents: [i16; 7], + pub entries: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum AlgebraSyntax { + Linear, + Affine(Reference), +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ExactSyntax { + pub value: Exact, + pub literal: LiteralSyntax, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct OverrideSyntax { + pub span: Span, + pub target: Reference, + pub binding: Binding, + pub binding_span: Span, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Body { + QuantityKind { + dimension: Field, + algebra: Field, + }, + Unit { + kind: Field, + scale: Field, + offset: Field, + symbol: Field, + }, + Calculation { + parameters: Vec, + result: Field, + body: Field, + }, + Property { + subject: Field, + value_type: Field, + base: Field, + display_unit: Option>>, + }, + Check { + subject: Option>>, + documented_constraint: Option>>, + predicate: Field, + }, + Case { + subject: Option>>, + base: Field<()>, + overrides: Field>, + }, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Declaration { + pub header: Header, + pub span: Span, + pub body_span: Span, + pub description: Option>, + /// Analysis roots may explicitly spell `parent none`; definitions may not. + pub parent: Option>, + pub body: Body, +} + +#[derive(Default)] +struct Fields { + dimension: Option>, + algebra: Option>, + kind: Option>, + scale: Option>, + offset: Option>, + symbol: Option>, + result: Option>, + body: Option>, + subject: Option>, + optional_subject: Option>>, + value_type: Option>, + base: Option>, + display_unit: Option>>, + documented_constraint: Option>>, + predicate: Option>, + baseline: Option>, + overrides: Option>>, +} + +fn required(value: Option, name: &str, at: Span) -> Result { + value.ok_or_else(|| error(at, format!("Missing required field '{name}'."))) +} + +impl Fields { + fn finish( + self, + kind: DeclarationKind, + parameters: Vec, + at: Span, + ) -> Result { + Ok(match kind { + DeclarationKind::QuantityKind => Body::QuantityKind { + dimension: required(self.dimension, "dimension", at)?, + algebra: required(self.algebra, "algebra", at)?, + }, + DeclarationKind::Unit => Body::Unit { + kind: required(self.kind, "kind", at)?, + scale: required(self.scale, "scale", at)?, + offset: required(self.offset, "offset", at)?, + symbol: required(self.symbol, "symbol", at)?, + }, + DeclarationKind::Calculation => Body::Calculation { + parameters, + result: required(self.result, "result", at)?, + body: required(self.body, "body", at)?, + }, + DeclarationKind::Property(_) => Body::Property { + subject: required(self.subject, "subject", at)?, + value_type: required(self.value_type, "type", at)?, + base: required(self.base, "base", at)?, + display_unit: self.display_unit, + }, + DeclarationKind::Check(_) => Body::Check { + subject: self.optional_subject, + documented_constraint: self.documented_constraint, + predicate: required(self.predicate, "predicate", at)?, + }, + DeclarationKind::Case(_) => Body::Case { + subject: self.optional_subject, + base: required(self.baseline, "base", at)?, + overrides: required(self.overrides, "overrides", at)?, + }, + }) + } +} + +/// Consume exactly one declaration from already charged full-file tokens. +/// `version` is inherited from the caller's entry policy; fragments have no +/// independent version header. Public parsing does not call this staged API. +pub(super) fn parse( + p: &mut Cursor<'_, '_>, + version: u32, + lexical_owner: Option, +) -> Result { + p.budget.active()?; + let kind_span = p.span(); + let kind = declaration_kind(p.text()) + .ok_or_else(|| error(kind_span, "Expected a short typed declaration word."))?; + if version != 8 { + return Err(error(kind_span, "Analysis declarations require source8.")); + } + if lexical_owner.is_some() { + return Err(error( + kind_span, + "Analysis declarations and definitions must be roots.", + )); + } + p.budget.entry(kind_span)?; + p.cursor += 1; + let alias = p.name()?; + let name_span = p.span(); + let name = p.quoted()?; + let identity = p.identity()?; + let opening = p.take("{")?; + let header = Header { + kind, + kind_span, + alias, + name, + name_span, + identity, + span: Span { + start: kind_span.start, + end: opening.start, + }, + }; + let mut fields = Fields::default(); + let mut description = None; + let mut parent = None; + let mut parameters = Vec::new(); + let mut seen = BTreeSet::new(); + let mut parameter_names = BTreeSet::new(); + let mut parameter_ids = BTreeSet::new(); + while p.text() != "}" { + let keyword = p.span(); + p.budget.entry(keyword)?; + if p.text() == "input" && kind == DeclarationKind::Calculation { + let parameter = p.parameter()?; + if !parameter_names.insert(parameter.name.name.clone()) { + return Err(error(parameter.name.span, "Duplicate parameter name.")); + } + if let Some(id) = parameter.identity.value + && !parameter_ids.insert(id) + { + return Err(error( + parameter.identity.literal.expect("explicit identity"), + "Duplicate parameter identity.", + )); + } + parameters.push(parameter); + continue; + } + // Borrow the original spelling for duplicate detection; no field-name + // allocation and no map insertion can overwrite an earlier value. + let source = p.source; + let key = &source[keyword.start..keyword.end]; + if !seen.insert(key) { + return Err(error(keyword, format!("Duplicate field '{key}'."))); + } + p.cursor += 1; + use DeclarationKind::*; + match (kind, key) { + (_, "description") => description = Some(p.field(keyword, |p| p.quoted())?), + (Property(_) | Check(_) | Case(_), "parent") => { + parent = Some(p.field(keyword, |p| { + p.take("none")?; + Ok(()) + })?) + } + (QuantityKind, "dimension") => { + fields.dimension = Some(p.field(keyword, |p| p.dimension())?) + } + (QuantityKind, "algebra") => fields.algebra = Some(p.field(keyword, |p| p.algebra())?), + (Unit, "kind") => fields.kind = Some(p.field(keyword, |p| p.at_reference())?), + (Unit, "scale") => fields.scale = Some(p.field(keyword, |p| p.exact_value())?), + (Unit, "offset") => fields.offset = Some(p.field(keyword, |p| p.exact_value())?), + (Unit, "symbol") => fields.symbol = Some(p.field(keyword, |p| p.quoted())?), + (Calculation, "result") => fields.result = Some(p.field(keyword, |p| p.value_type())?), + (Calculation, "body") => fields.body = Some(p.field(keyword, syntax::binding)?), + (Property(_), "subject") => { + fields.subject = Some(p.field(keyword, |p| p.at_reference())?) + } + (Property(_), "type") => { + fields.value_type = Some(p.field(keyword, |p| p.value_type())?) + } + (Property(_), "base") => fields.base = Some(p.field(keyword, syntax::binding)?), + (Property(_), "display_unit") => { + fields.display_unit = Some(p.field(keyword, |p| p.optional_reference())?) + } + (Check(_) | Case(_), "subject") => { + fields.optional_subject = Some(p.field(keyword, |p| p.optional_reference())?) + } + (Check(_), "documented_constraint") => { + fields.documented_constraint = Some(p.field(keyword, |p| p.optional_reference())?) + } + (Check(_), "predicate") => fields.predicate = Some(p.field(keyword, syntax::binding)?), + (Case(_), "base") => { + fields.baseline = Some(p.field(keyword, |p| { + p.take("ModelBaseline")?; + Ok(()) + })?) + } + (Case(_), "overrides") => fields.overrides = Some(p.field(keyword, |p| p.overrides())?), + _ => { + return Err(error( + keyword, + format!("Unknown field '{key}' for this typed declaration."), + )); + } + } + } + let closing = p.take("}")?; + let body = fields.finish(kind, parameters, closing)?; + Ok(Declaration { + header, + span: Span { + start: kind_span.start, + end: closing.end, + }, + body_span: Span { + start: opening.end, + end: closing.start, + }, + description, + parent, + body, + }) +} + +impl Cursor<'_, '_> { + fn field( + &mut self, + keyword: Span, + parse: impl FnOnce(&mut Self) -> Result, + ) -> Result, Error> { + let start = self.span().start; + let value = parse(self)?; + let end = self.last_end(); + let terminator = self.take(";")?; + Ok(Field { + span: Span { + start: keyword.start, + end: terminator.end, + }, + keyword, + value_span: Span { start, end }, + value, + }) + } + + fn identity(&mut self) -> Result { + if self.text() != "id" { + return Ok(IdentitySyntax { + value: None, + keyword: None, + literal: None, + insertion: Some(self.span().start), + }); + } + let keyword = self.take("id")?; + let literal = self.span(); + let spelling = self.quoted()?; + if spelling.len() != 36 + || ![8, 13, 18, 23] + .into_iter() + .all(|at| spelling.as_bytes()[at] == b'-') + { + return Err(error( + literal, + "Expected UUID in 8-4-4-4-12 hexadecimal form.", + )); + } + let id = Id::parse_str(&spelling).map_err(|_| error(literal, "Invalid UUID identity."))?; + Ok(IdentitySyntax { + value: Some(id), + keyword: Some(keyword), + literal: Some(literal), + insertion: None, + }) + } + + fn parameter(&mut self) -> Result { + let start = self.take("input")?.start; + let name = self.name()?; + let identity = self.identity()?; + self.take(":")?; + let value_type = self.value_type()?; + let end = self.take(";")?.end; + Ok(ParameterSyntax { + span: Span { start, end }, + name, + identity, + value_type, + }) + } + + fn at_reference(&mut self) -> Result { + self.take("@")?; + self.reference() + } + + fn optional_reference(&mut self) -> Result, Error> { + if self.text() == "none" { + self.cursor += 1; + Ok(None) + } else { + self.at_reference().map(Some) + } + } + + fn exact_value(&mut self) -> Result { + let (value, literal) = self.exact_literal()?; + Ok(ExactSyntax { value, literal }) + } + + fn value_type(&mut self) -> Result { + let start = self.span().start; + self.budget.entry(self.span())?; + let list = self.text() == "List"; + if list { + self.open()?; + self.cursor += 1; + self.take("(")?; + } + let scalar = self.scalar_type()?; + if list { + self.take(")")?; + self.close(); + } + Ok(TypeSyntax { + span: Span { + start, + end: self.last_end(), + }, + scalar, + list, + }) + } + + fn scalar_type(&mut self) -> Result { + let start = self.span().start; + let kind = match self.text() { + "Boolean" => { + self.cursor += 1; + ScalarKind::Boolean + } + "Number" => { + self.cursor += 1; + ScalarKind::Number + } + "Quantity" => { + self.open()?; + self.cursor += 1; + self.take("(")?; + let reference = self.at_reference()?; + self.take(")")?; + self.close(); + ScalarKind::Quantity(reference) + } + _ => { + return Err(error( + self.span(), + "Expected Boolean, Number or Quantity(@kind); nested List is not a scalar type.", + )); + } + }; + Ok(ScalarSyntax { + span: Span { + start, + end: self.last_end(), + }, + kind, + }) + } + + fn dimension(&mut self) -> Result { + self.take("{")?; + let mut exponents = [0; 7]; + let mut seen = [false; 7]; + let mut entries = Vec::new(); + while self.text() != "}" { + let axis = self.span(); + self.budget.entry(axis)?; + let index = match self.text() { + "length" => 0, + "mass" => 1, + "time" => 2, + "electric_current" => 3, + "temperature" => 4, + "amount" => 5, + "luminous_intensity" => 6, + _ => return Err(error(axis, "Unknown dimension axis.")), + }; + if seen[index] { + return Err(error(axis, "Duplicate dimension axis.")); + } + seen[index] = true; + self.cursor += 1; + let start = self.span().start; + let negative = self.text() == "-"; + if negative { + self.cursor += 1; + } + let digits = self.integer()?; + let value = self.source[digits.start..digits.end] + .parse::() + .map_err(|_| error(digits, "Dimension exponent exceeds -32..32."))?; + if value > 32 { + return Err(error(digits, "Dimension exponent exceeds -32..32.")); + } + exponents[index] = if negative { -value } else { value }; + let exponent = Span { + start, + end: digits.end, + }; + let end = self.take(";")?.end; + entries.push(DimensionEntry { + axis, + exponent, + span: Span { + start: axis.start, + end, + }, + }); + } + self.take("}")?; + Ok(DimensionSyntax { exponents, entries }) + } + + fn algebra(&mut self) -> Result { + if self.text() == "Linear" { + self.cursor += 1; + return Ok(AlgebraSyntax::Linear); + } + self.take("Affine")?; + self.open()?; + self.take("(")?; + let difference = self.at_reference()?; + self.take(")")?; + self.close(); + Ok(AlgebraSyntax::Affine(difference)) + } + + fn overrides(&mut self) -> Result, Error> { + self.take("{")?; + let mut entries = Vec::new(); + let mut names = BTreeSet::new(); + while self.text() != "}" { + let start = self.span().start; + self.budget.entry(self.span())?; + let target = self.at_reference()?; + if !names.insert(target.name.clone()) { + return Err(error(target.span, "Duplicate override target.")); + } + self.take("=")?; + let binding_start = self.span().start; + let binding = syntax::binding(self)?; + let binding_span = Span { + start: binding_start, + end: self.last_end(), + }; + let end = self.take(";")?.end; + entries.push(OverrideSyntax { + span: Span { start, end }, + target, + binding, + binding_span, + }); + } + self.take("}")?; + Ok(entries) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/oa-core/src/language/analysis_source/declarations/tests.rs b/crates/oa-core/src/language/analysis_source/declarations/tests.rs new file mode 100644 index 00000000..3d739c8f --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/declarations/tests.rs @@ -0,0 +1,531 @@ +use super::*; +use crate::language::ParseLimits; +use crate::language::analysis_source::budget::Limits; + +fn limits() -> Limits { + Limits::source8(ParseLimits::UNLIMITED) +} +fn budget() -> Budget { + Budget::new(limits()).unwrap() +} + +fn declaration(source: &str) -> Declaration { + let mut budget = budget(); + let tokens = lex::tokens(source, &mut budget).unwrap(); + let mut cursor = Cursor::new(source, &tokens, &mut budget); + let result = parse(&mut cursor, 8, None).unwrap(); + cursor.end().unwrap(); + result +} + +fn failure(source: &str) -> Error { + let mut budget = budget(); + let tokens = lex::tokens(source, &mut budget).unwrap(); + parse(&mut Cursor::new(source, &tokens, &mut budget), 8, None).unwrap_err() +} + +fn text(source: &str, span: Span) -> &str { + &source[span.start..span.end] +} + +#[test] +fn required_declarations_have_independent_typed_body_and_header_oracles() { + let source = "quantity_kind duration \"Duration\" id \"188b911c-5f8b-431c-9eea-2d41f3953767\" { dimension { time 1; length -2; }; algebra Affine(@difference); description \"雪\"; }"; + let actual = declaration(source); + assert_eq!(actual.header.kind, DeclarationKind::QuantityKind); + assert_eq!(actual.header.alias.name, "duration"); + assert_eq!(actual.header.name, "Duration"); + assert_eq!( + actual.header.identity.value, + Some(Id::parse_str("188b911c-5f8b-431c-9eea-2d41f3953767").unwrap()) + ); + assert_eq!(actual.description.as_ref().unwrap().value, "雪"); + assert!(actual.parent.is_none()); + let Body::QuantityKind { dimension, algebra } = actual.body else { + panic!("quantity kind") + }; + assert_eq!(dimension.value.exponents, [-2, 0, 1, 0, 0, 0, 0]); + assert_eq!(text(source, dimension.keyword), "dimension"); + assert_eq!(text(source, dimension.value_span), "{ time 1; length -2; }"); + assert_eq!(text(source, dimension.value.entries[1].exponent), "-2"); + let AlgebraSyntax::Affine(difference) = algebra.value else { + panic!("affine") + }; + assert_eq!(difference.name, "difference"); + + let Body::Unit { kind, scale, offset, symbol } = declaration("unit millisecond \"Millisecond\" { kind @duration; scale ratio (1, // divisor\n 1000); offset 0; symbol \"ms\"; }").body else { panic!("unit") }; + assert_eq!(kind.value.name, "duration"); + assert_eq!(scale.value.value.canonical_literal(), "ratio(1, 1000)"); + assert_eq!(offset.value.value.canonical_literal(), "0"); + assert_eq!(symbol.value, "ms"); + + let Body::Calculation { parameters, result, body } = declaration("calculation combine \"Combine durations\" { input a id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Quantity(@duration); input b id \"7a47cc1e-8ae5-4551-b2d8-8a68108e0976\" : Quantity(@duration); result Quantity(@duration); body $a + $b; }").body else { panic!("calculation") }; + assert_eq!( + parameters + .iter() + .map(|p| p.name.name.as_str()) + .collect::>(), + ["a", "b"] + ); + assert!(!result.value.list); + assert!( + matches!(result.value.scalar.kind, ScalarKind::Quantity(ref r) if r.name == "duration") + ); + let Binding::Expression(Expression { + kind: ExpressionKind::Binary { op, left, right }, + .. + }) = body.value + else { + panic!("binary") + }; + assert_eq!(op, BinaryOp::Add); + assert!(matches!(left.kind, ExpressionKind::Parameter(ref r) if r.name == "a")); + assert!(matches!(right.kind, ExpressionKind::Parameter(ref r) if r.name == "b")); + + for (word, layer) in [("oa_property", Layer::Oa), ("sa_property", Layer::Sa)] { + let actual = declaration(&format!( + "{word} receive_time \"Receive time\" {{ subject @receive; type Quantity(@duration); base 750 [@millisecond]; display_unit @millisecond; parent none; }}" + )); + assert_eq!(actual.header.kind, DeclarationKind::Property(layer)); + assert!(actual.parent.is_some()); + let Body::Property { + subject, + value_type, + base, + display_unit, + } = actual.body + else { + panic!("property") + }; + assert_eq!(subject.value.name, "receive"); + assert!(matches!( + value_type.value.scalar.kind, + ScalarKind::Quantity(_) + )); + assert!(matches!( + base.value, + Binding::Expression(Expression { + kind: ExpressionKind::Quantity { .. }, + .. + }) + )); + assert_eq!(display_unit.unwrap().value.unwrap().name, "millisecond"); + } + for word in ["oa_check", "sa_check"] { + let Body::Check { subject, documented_constraint, predicate } = declaration(&format!("{word} budget \"Response budget\" {{ subject none; documented_constraint @constraint; predicate @total_time <= @limit; }}")).body else { panic!("check") }; + assert!(subject.unwrap().value.is_none()); + assert_eq!( + documented_constraint.unwrap().value.unwrap().name, + "constraint" + ); + assert!(matches!( + predicate.value, + Binding::Expression(Expression { + kind: ExpressionKind::Binary { + op: BinaryOp::LessOrEqual, + .. + }, + .. + }) + )); + } + for word in ["oa_analysis", "sa_analysis"] { + let Body::Case { subject, base, overrides } = declaration(&format!("{word} degraded \"Degraded radio\" {{ subject @respond; base ModelBaseline; overrides {{ @receive_time = unbound; }}; }}")).body else { panic!("case") }; + assert_eq!(subject.unwrap().value.unwrap().name, "respond"); + assert!(base.value_span.end > base.value_span.start); + assert_eq!(overrides.value.len(), 1); + assert_eq!(overrides.value[0].target.name, "receive_time"); + assert!(matches!(overrides.value[0].binding, Binding::Unbound(_))); + } +} + +#[test] +fn empty_unbound_and_all_scalar_list_types_are_explicit_valid_syntax() { + for (spelling, list, kind) in [ + ("Boolean", false, 0), + ("Number", false, 1), + ("Quantity(@duration)", false, 2), + ("List(Boolean)", true, 0), + ("List(Number)", true, 1), + ("List(Quantity(@duration))", true, 2), + ] { + let Body::Calculation { + parameters, + result, + body, + } = declaration(&format!( + "calculation c \"C\" {{ result {spelling}; body unbound; }}" + )) + .body + else { + panic!() + }; + assert!(parameters.is_empty()); + assert_eq!(result.value.list, list); + assert_eq!( + match result.value.scalar.kind { + ScalarKind::Boolean => 0, + ScalarKind::Number => 1, + ScalarKind::Quantity(_) => 2, + }, + kind + ); + assert!(matches!(body.value, Binding::Unbound(_))); + } + let Body::QuantityKind { dimension, algebra } = + declaration("quantity_kind q \"Q\" { dimension {}; algebra Linear; }").body + else { + panic!() + }; + assert_eq!(dimension.value.exponents, [0; 7]); + assert!(dimension.value.entries.is_empty()); + assert_eq!(algebra.value, AlgebraSyntax::Linear); + let Body::Case { + subject, overrides, .. + } = declaration("oa_analysis c \"C\" { base ModelBaseline; overrides {}; }").body + else { + panic!() + }; + assert!(subject.is_none()); + assert!(overrides.value.is_empty()); + let Body::Check { + subject, + documented_constraint, + .. + } = declaration("oa_check c \"C\" { predicate unbound; }").body + else { + panic!() + }; + assert!(subject.is_none() && documented_constraint.is_none()); + let Body::Property { display_unit, .. } = + declaration("oa_property p \"P\" { subject @s; type Number; base unbound; }").body + else { + panic!() + }; + assert!(display_unit.is_none()); +} + +#[test] +fn missing_required_fields_and_unknown_or_duplicate_entries_reject_locally() { + let required_sets: &[(&str, &[&str])] = &[ + ( + "quantity_kind q \"Q\"", + &["dimension {};", "algebra Linear;"], + ), + ( + "unit u \"U\"", + &["kind @q;", "scale 1;", "offset 0;", "symbol \"u\";"], + ), + ("calculation c \"C\"", &["result Number;", "body unbound;"]), + ( + "oa_property p \"P\"", + &["subject @s;", "type Number;", "base unbound;"], + ), + ("oa_check c \"C\"", &["predicate unbound;"]), + ( + "oa_analysis a \"A\"", + &["base ModelBaseline;", "overrides {};"], + ), + ]; + for (header, fields) in required_sets { + declaration(&format!("{header} {{ {} }}", fields.join(" "))); + for removed in 0..fields.len() { + let remaining = fields + .iter() + .enumerate() + .filter_map(|(i, field)| (i != removed).then_some(*field)) + .collect::>() + .join(" "); + let error = failure(&format!("{header} {{ {remaining} }}")); + assert!( + error.message.contains("Missing required field"), + "{error:?}" + ); + let duplicate = format!("{header} {{ {} {} }}", fields.join(" "), fields[removed]); + assert!(failure(&duplicate).message.contains("Duplicate field")); + } + } + for source in [ + "quantity_kind q \"Q\" { dimension { time 1; time 2; }; algebra Linear; }", + "quantity_kind q \"Q\" { dimension { unknown 1; }; algebra Linear; }", + "quantity_kind q \"Q\" { dimension { time 33; }; algebra Linear; }", + "unit u \"U\" { kind @q; scale 1 + 2; offset 0; symbol \"u\"; }", + "unit u \"U\" { kind @q; scale 1 [@u]; offset 0; symbol \"u\"; }", + "unit u \"U\" { kind @q; scale ratio(1, 0); offset 0; symbol \"u\"; }", + "calculation c \"C\" { result List(List(Number)); body unbound; }", + "calculation c \"C\" { input x : Number; input x : Number; result Number; body unbound; }", + "calculation c \"C\" { input x id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Number; input y id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Number; result Number; body unbound; }", + "oa_property p \"P\" { subject none; type Number; base unbound; }", + "oa_property p \"P\" { subject @s; type Number; base none; }", + "oa_property p \"P\" { subject @s; type Number; base unbound; parent @s; }", + "oa_analysis a \"A\" { base @another; overrides {}; }", + "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = clear; }; }", + "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = 1; @p = 2; }; }", + "oa_check c \"C\" { predicate @f(x: 1, x: 2); }", + "quantity_kind q \"Q\" { dimension {}; algebra Linear; parent none; }", + "oa_check c \"C\" { predicate unbound; invented none; }", + ] { + failure(source); + } + let duplicate = "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = 1; @p = 2; }; }"; + let error = failure(duplicate); + assert_eq!(text(duplicate, error.span), "p"); + assert!(error.span.start > duplicate.find("@p").unwrap() + 1); +} + +#[test] +fn borrowed_cursor_preserves_full_file_spans_and_does_not_rescan_fields() { + let source = "// 雪\r\ncalculation c \"C\" { input a : Number; result Number; body @combine(a: ratio (1, // first\r\n 2), b: $a) + @combine(a: 2, b: $a); }\r\nunit u \"U\" { kind @q; scale 1; offset 0; symbol \"u\"; }"; + let mut shared = budget(); + let tokens = lex::tokens(source, &mut shared).unwrap(); + let lexical = shared.usage; + let mut cursor = Cursor::new(source, &tokens, &mut shared); + let first = parse(&mut cursor, 8, None).unwrap(); + assert_eq!(cursor.text(), "unit"); + let first_end = first.span.end; + let second = parse(&mut cursor, 8, None).unwrap(); + cursor.end().unwrap(); + assert!(second.span.start > first_end); + assert_eq!(text(source, first.header.kind_span), "calculation"); + assert_eq!(text(source, first.header.name_span), "\"C\""); + assert_eq!(text(source, second.header.alias.span), "u"); + assert_eq!( + text(source, first.body_span).trim().strip_prefix("input"), + Some( + " a : Number; result Number; body @combine(a: ratio (1, // first\r\n 2), b: $a) + @combine(a: 2, b: $a);" + ) + ); + let Body::Calculation { + parameters, body, .. + } = first.body + else { + panic!() + }; + assert_eq!(text(source, parameters[0].span), "input a : Number;"); + assert_eq!(text(source, parameters[0].value_type.span), "Number"); + let Binding::Expression(Expression { + kind: ExpressionKind::Binary { left, right, .. }, + .. + }) = body.value + else { + panic!() + }; + let ExpressionKind::Call { + arguments: left, .. + } = left.kind + else { + panic!() + }; + let ExpressionKind::Call { + arguments: right, .. + } = right.kind + else { + panic!() + }; + let spans = [ + left[0].parameter.span, + left[1].parameter.span, + right[0].parameter.span, + right[1].parameter.span, + ]; + assert_eq!(spans.map(|span| text(source, span)), ["a", "b", "a", "b"]); + assert!(spans.windows(2).all(|pair| pair[0].end < pair[1].start)); + let literal = left[0].value.literal.as_ref().unwrap(); + assert_eq!(text(source, literal.span), "ratio (1, // first\r\n 2)"); + assert_eq!( + literal + .tokens + .iter() + .map(|span| text(source, *span)) + .collect::>(), + ["ratio", "(", "1", ",", "2", ")"] + ); + assert_eq!(shared.usage.bytes, source.len()); + assert_eq!(shared.usage.bytes, lexical.bytes); + assert_eq!(shared.usage.tokens, lexical.tokens); + assert_eq!(shared.usage.references, 9); // two calls, four keys, two $a, unit kind. +} + +#[test] +fn missing_id_metadata_is_stable_and_parameter_ids_are_owner_scoped() { + let source = + "calculation c \"C\" { input a // identity here\r\n : Number; result Number; body $a; }"; + let first = declaration(source); + let second = declaration(source); + assert_eq!(first, second); + assert_eq!( + first.header.identity.insertion, + Some(source.find('{').unwrap()) + ); + let Body::Calculation { parameters, .. } = first.body else { + panic!() + }; + let id = ¶meters[0].identity; + assert!(id.value.is_none() && id.literal.is_none() && id.keyword.is_none()); + assert_eq!(id.insertion, Some(source.find(':').unwrap())); + let explicit = "28ff40b0-1b8f-4735-8536-5a6ab21b56cb"; + // Syntax does not add ParameterIds to a global identity registry, and an + // explicit parameter ID can equal its own calculation's global UUID. + for name in ["one", "two"] { + let source = format!( + "calculation {name} \"C\" id \"{explicit}\" {{ input a id \"{explicit}\" : Number; result Number; body $a; }}" + ); + let actual = declaration(&source); + let Body::Calculation { parameters, .. } = actual.body else { + panic!() + }; + assert_eq!(parameters[0].identity.value, actual.header.identity.value); + assert!(parameters[0].identity.insertion.is_none()); + } +} + +#[test] +fn inherited_version_root_and_short_word_contracts_do_not_enable_public8() { + let source = "oa_check c \"C\" { predicate unbound; }"; + for version in [1, 2, 3, 4, 5, 6, 7, 9] { + let mut budget = budget(); + let tokens = lex::tokens(source, &mut budget).unwrap(); + let error = parse( + &mut Cursor::new(source, &tokens, &mut budget), + version, + None, + ) + .unwrap_err(); + assert_eq!(text(source, error.span), "oa_check"); + assert!(error.message.contains("source8")); + } + let mut budget = budget(); + let tokens = lex::tokens(source, &mut budget).unwrap(); + assert!( + parse( + &mut Cursor::new(source, &tokens, &mut budget), + 8, + Some(Id::nil()) + ) + .unwrap_err() + .message + .contains("roots") + ); + for word in [ + "oa_engineering_property", + "sa_engineering_property", + "oa_analysis_check", + "sa_analysis_check", + "oa_analysis_case", + "sa_analysis_case", + ] { + assert!(declaration_kind(word).is_none()); + } + for alias in [ + "unit", + "calculation", + "quantity_kind", + "oa_property", + "if", + "and", + "List", + ] { + let old = format!("arcadia 7; project {alias} \"P\" {{}}"); + assert!(crate::language::parse(&old).is_ok(), "{old}"); + } + let contextual = + declaration("calculation unit \"C\" { input and : Number; result Number; body $and; }"); + assert_eq!(contextual.header.alias.name, "unit"); + assert!(crate::language::parse("arcadia 8; project p \"P\" {}").is_err()); + for version in 3..=7 { + assert!( + crate::language::parse(&format!( + "arcadia {version}; project p \"P\" {{ {source} }}" + )) + .is_err() + ); + } +} + +#[test] +fn declaration_budgets_share_literals_types_references_and_failure_state() { + let source = "unit u \"U\" { kind @q; scale ratio(1, 3); offset ratio(2, 7); symbol \"u\"; }"; + let mut measured = budget(); + let tokens = lex::tokens(source, &mut measured).unwrap(); + parse(&mut Cursor::new(source, &tokens, &mut measured), 8, None).unwrap(); + let numeric = measured.numeric_units(); + assert!(numeric > 0); + // All declaration, field and type entries share the same preallocation + // ceiling. Measure a deterministic valid counterpart before reducing it. + for (maximum, succeeds) in [ + (measured.usage.entries, true), + (measured.usage.entries - 1, false), + ] { + let mut profile = limits(); + profile.entries = maximum; + let mut shared = Budget::new(profile).unwrap(); + let tokens = lex::tokens(source, &mut shared).unwrap(); + let result = parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None); + assert_eq!(result.is_ok(), succeeds); + if !succeeds { + assert!(result.unwrap_err().message.contains("entries")); + } + } + for (maximum, succeeds) in [(numeric, true), (numeric - 1, false)] { + let mut profile = limits(); + profile.numeric_units = maximum; + let mut shared = Budget::new(profile).unwrap(); + let tokens = lex::tokens(source, &mut shared).unwrap(); + let result = parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None); + assert_eq!(result.is_ok(), succeeds); + } + let first = + "calculation c \"C\" { input x : Quantity(@q); result Quantity(@q); body unbound; }"; + let second = "oa_property p \"P\" { subject @s; type Quantity(@q); base 1 [@u]; }"; + let mut profile = limits(); + profile.references = 4; + let mut shared = Budget::new(profile).unwrap(); + let tokens = lex::tokens(first, &mut shared).unwrap(); + parse(&mut Cursor::new(first, &tokens, &mut shared), 8, None).unwrap(); + assert_eq!(shared.usage.references, 2); + let tokens = lex::tokens(second, &mut shared).unwrap(); + let error = parse(&mut Cursor::new(second, &tokens, &mut shared), 8, None).unwrap_err(); + assert_eq!(text(second, error.span), "u"); + assert_eq!(shared.usage.references, 4); + assert_eq!(shared.active().unwrap_err(), error); + let numeric_after_failure = shared.numeric_units(); + assert!(lex::tokens("1", &mut shared).is_err()); + assert_eq!(shared.numeric_units(), numeric_after_failure); + let unbound_tokens = lex::tokens("unbound", &mut budget()).unwrap(); + assert_eq!( + syntax::binding(&mut Cursor::new("unbound", &unbound_tokens, &mut shared)).unwrap_err(), + error + ); + assert!( + declaration("oa_check fresh \"Fresh\" { predicate true; }") + .parent + .is_none() + ); +} + +#[test] +fn typed_frame_budget_and_truncated_declaration_cleanup_have_nearby_successes() { + let source = "calculation c \"C\" { result List(Quantity(@q)); body unbound; }"; + for (maximum, succeeds) in [(2, true), (1, false)] { + let mut profile = limits(); + profile.frames = maximum; + let mut shared = Budget::new(profile).unwrap(); + let tokens = lex::tokens(source, &mut shared).unwrap(); + assert_eq!( + parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None).is_ok(), + succeeds + ); + } + let source = "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = if(true, ratio(1, 2), @f(a: 1)); }; }"; + let tokens = lex::tokens(source, &mut budget()).unwrap(); + for token in tokens + .iter() + .filter(|token| matches!(text(source, token.span), "{" | "(" | "," | ":" | "=")) + { + let prefix = &source[..token.span.end]; + let mut shared = budget(); + let tokens = lex::tokens(prefix, &mut shared).unwrap(); + assert!(parse(&mut Cursor::new(prefix, &tokens, &mut shared), 8, None).is_err()); + declaration("oa_check fresh \"Fresh\" { predicate true; }"); + } +} diff --git a/crates/oa-core/src/language/analysis_source/lex.rs b/crates/oa-core/src/language/analysis_source/lex.rs index 0629d7cb..335d47c5 100644 --- a/crates/oa-core/src/language/analysis_source/lex.rs +++ b/crates/oa-core/src/language/analysis_source/lex.rs @@ -1,4 +1,5 @@ -//! Raw numeric tokens: finite coordinate checks remain in the existing reader. +//! Source8 significant-token adapter over the shared raw scanner. +use super::super::raw_tokens::{Class as RawClass, Scanner}; use super::{Budget, Error, Span, error}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -8,11 +9,13 @@ pub enum Class { String, Punctuation, } + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Token { pub class: Class, pub span: Span, } + pub fn tokens(source: &str, budget: &mut Budget) -> Result, Error> { budget.bytes( source.len(), @@ -21,118 +24,29 @@ pub fn tokens(source: &str, budget: &mut Budget) -> Result, Error> { end: source.len(), }, )?; - let bytes = source.as_bytes(); - let mut cursor = 0; + let mut scanner = Scanner::new(source); let mut result = Vec::new(); - while cursor < bytes.len() { - let start = cursor; - // Reserve before scanning/storing a token, including skipped trivia. - budget.token(Span { start, end: start })?; - let class = if bytes[cursor].is_ascii_whitespace() { - while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { - cursor += 1; - } - None - } else if bytes[cursor..].starts_with(b"//") { - while cursor < bytes.len() && bytes[cursor] != b'\n' { - cursor += 1; - } - None - } else if bytes[cursor].is_ascii_alphabetic() || bytes[cursor] == b'_' { - cursor += 1; - while cursor < bytes.len() - && (bytes[cursor].is_ascii_alphanumeric() || bytes[cursor] == b'_') - { - cursor += 1; - } - Some(Class::Word) - } else if bytes[cursor].is_ascii_digit() { - while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { - cursor += 1; - } - if bytes.get(cursor) == Some(&b'.') { - cursor += 1; - let fraction = cursor; - while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { - cursor += 1; - } - if cursor == fraction { - return Err(error( - Span { start, end: cursor }, - "Expected decimal fraction digits.", - )); - } - } - if matches!(bytes.get(cursor), Some(b'e' | b'E')) { - cursor += 1; - if matches!(bytes.get(cursor), Some(b'+' | b'-')) { - cursor += 1; - } - let exponent = cursor; - while cursor < bytes.len() && bytes[cursor].is_ascii_digit() { - cursor += 1; - } - if cursor == exponent { - return Err(error( - Span { start, end: cursor }, - "Expected exponent digits.", - )); - } - } - if bytes - .get(cursor) - .is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.')) - { - return Err(error( - Span { start, end: cursor }, - "Malformed numeric atom.", - )); - } - Some(Class::Number) - } else if bytes[cursor] == b'"' { - cursor += 1; - let mut closed = false; - while cursor < bytes.len() { - match bytes[cursor] { - b'\\' => { - cursor += 1; - if cursor < bytes.len() { - cursor += 1; - } - } - b'"' => { - cursor += 1; - closed = true; - break; - } - _ => cursor += 1, - } - } - if !closed { - return Err(error(Span { start, end: cursor }, "Unclosed string.")); - } - // Quoted field decoding remains the typed declaration consumer's job. - Some(Class::String) - } else if b"{}();,[]:=@$+-*/<>!".contains(&bytes[cursor]) { - cursor += 1; - if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') && bytes.get(cursor) == Some(&b'=') - { - cursor += 1; - } - Some(Class::Punctuation) - } else { - let end = start + source[start..].chars().next().unwrap().len_utf8(); - return Err(error( - Span { start, end }, - "Unexpected expression character.", - )); + let mut offset = 0; + while offset < source.len() { + // Charge trivia too, before the scanner advances or a token is stored. + budget.token(Span { + start: offset, + end: offset, + })?; + let token = scanner.next().expect("source remains"); + offset = token.span.end; + let class = match token.class { + RawClass::Word => Class::Word, + RawClass::Number => Class::Number, + RawClass::String => Class::String, + RawClass::Punctuation => Class::Punctuation, + RawClass::Whitespace | RawClass::Comment => continue, + RawClass::Error => return Err(error(token.span, "Invalid source token.")), }; - if let Some(class) = class { - result.push(Token { - class, - span: Span { start, end: cursor }, - }); - } + result.push(Token { + class, + span: token.span, + }); } Ok(result) } diff --git a/crates/oa-core/src/language/analysis_source/syntax.rs b/crates/oa-core/src/language/analysis_source/syntax.rs index 489bb1bb..70ac42b7 100644 --- a/crates/oa-core/src/language/analysis_source/syntax.rs +++ b/crates/oa-core/src/language/analysis_source/syntax.rs @@ -1,6 +1,7 @@ //! Iterative Pratt machine. Source nesting does not recurse on the native stack. +use super::cursor::Cursor; use super::*; -use lex::{Class, Token}; +use lex::Class; use std::collections::BTreeSet; enum SequenceKind { @@ -27,25 +28,11 @@ enum Task { Power(usize), AsKind(usize, Reference), } -struct Parser<'a, 'b> { - source: &'a str, - tokens: &'a [Token], - cursor: usize, - frames: usize, - budget: &'b mut Budget, -} -pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Result { - let mut p = Parser { - source, - tokens, - cursor: 0, - frames: 0, - budget, - }; +pub(super) fn binding(p: &mut Cursor<'_, '_>) -> Result { + p.budget.active()?; if p.text() == "unbound" { let span = p.span(); p.cursor += 1; - p.end()?; return Ok(Binding::Unbound(span)); } let mut tasks = vec![Task::Expression(0)]; @@ -109,7 +96,7 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu ); } "ratio" => { - let (value, literal) = p.ratio()?; + let (value, literal) = p.exact_literal()?; values.push(p.number(value, literal)?); } "abs" | "not" | "if" | "sum" => { @@ -146,16 +133,8 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu .get(p.cursor) .is_some_and(|token| token.class == Class::Number) => { - let span = p.span(); - let value = p.budget.exact(&p.source[span.start..span.end], span)?; - p.cursor += 1; - values.push(p.number( - value, - LiteralSyntax { - span, - tokens: vec![span], - }, - )?); + let (value, literal) = p.exact_literal()?; + values.push(p.number(value, literal)?); } _ => return Err(error(p.span(), "Expected an analysis expression.")), } @@ -317,7 +296,6 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu } } } - p.end()?; Ok(Binding::Expression( values.pop().expect("one complete expression"), )) @@ -332,110 +310,7 @@ impl Sequence { } } } -impl Parser<'_, '_> { - fn span(&self) -> Span { - self.tokens.get(self.cursor).map_or( - Span { - start: self.source.len(), - end: self.source.len(), - }, - |token| token.span, - ) - } - fn text(&self) -> &str { - let span = self.span(); - &self.source[span.start..span.end] - } - fn take(&mut self, expected: &str) -> Result { - if self.text() != expected { - return Err(error(self.span(), format!("Expected '{expected}'."))); - } - let span = self.span(); - self.cursor += 1; - Ok(span) - } - fn end(&self) -> Result<(), Error> { - if self.cursor != self.tokens.len() { - Err(error( - self.span(), - "Unexpected content after analysis expression.", - )) - } else { - Ok(()) - } - } - fn open(&mut self) -> Result<(), Error> { - self.budget.frame(self.frames + 1, self.span())?; - self.frames += 1; - Ok(()) - } - fn close(&mut self) { - self.frames -= 1; - } - fn reference(&mut self) -> Result { - let span = self.span(); - if !self - .tokens - .get(self.cursor) - .is_some_and(|token| token.class == Class::Word) - || !crate::valid_alias(self.text()) - { - return Err(error(span, "Expected a valid reference or parameter name.")); - } - self.budget.reference(span)?; - let name = self.text().to_owned(); - self.cursor += 1; - Ok(Reference { name, span }) - } - fn integer(&mut self) -> Result { - let span = self.span(); - let text = self.text(); - if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { - return Err(error(span, "Expected unsigned integer digits.")); - } - if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS { - return Err(error(span, "Integer component exceeds digit limit.")); - } - self.cursor += 1; - Ok(span) - } - fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> { - let first_token = self.cursor; - let start = self.take("ratio")?.start; - self.open()?; - self.take("(")?; - let negative = if self.text() == "-" { - self.cursor += 1; - true - } else { - false - }; - let numerator = self.integer()?; - self.take(",")?; - let denominator = self.integer()?; - let end = self.take(")")?.end; - self.close(); - let n = &self.source[numerator.start..numerator.end]; - let d = &self.source[denominator.start..denominator.end]; - let length = 9 + usize::from(negative) + n.len() + d.len(); - if length > crate::analysis_numeric::MAX_LITERAL_BYTES { - return Err(error( - Span { start, end }, - "Numeric reconstruction exceeds literal byte limit.", - )); - } - let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" }); - Ok(( - self.budget.exact(&normalized, Span { start, end })?, - LiteralSyntax { - span: Span { start, end }, - tokens: self.tokens[first_token..self.cursor] - .iter() - .map(|token| token.span) - .collect(), - }, - )) - } +impl Cursor<'_, '_> { fn number(&mut self, magnitude: Exact, literal: LiteralSyntax) -> Result { let Span { start, end } = literal.span; let mut expression = if self.text() == "[" { diff --git a/crates/oa-core/src/language/raw_tokens.rs b/crates/oa-core/src/language/raw_tokens.rs new file mode 100644 index 00000000..d361cd3f --- /dev/null +++ b/crates/oa-core/src/language/raw_tokens.rs @@ -0,0 +1,347 @@ +//! One raw lexical scan for legacy adapters and staged typed source syntax. +//! Numbers remain source spans; only consumers choose a numeric domain. +use super::Span; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Class { + Word, + Number, + String, + Punctuation, + Whitespace, + Comment, + Error, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct Token { + pub class: Class, + pub span: Span, + /// Entire token belongs to the old greedy numeric-token alphabet. + pub legacy_number_part: bool, +} + +pub(super) struct Scanner<'a> { + source: &'a str, + cursor: usize, +} + +impl<'a> Scanner<'a> { + pub fn new(source: &'a str) -> Self { + Self { source, cursor: 0 } + } +} + +impl Iterator for Scanner<'_> { + type Item = Token; + + fn next(&mut self) -> Option { + let source = self.source; + let bytes = source.as_bytes(); + let start = self.cursor; + let first = *bytes.get(start)?; + let mut legacy_number_part = false; + let class = if matches!(first, b' ' | b'\t' | b'\r' | b'\n') { + self.cursor += 1; + while bytes + .get(self.cursor) + .is_some_and(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n')) + { + self.cursor += 1; + } + Class::Whitespace + } else if bytes[start..].starts_with(b"//") { + self.cursor += 2; + while bytes.get(self.cursor).is_some_and(|b| *b != b'\n') { + self.cursor += 1; + } + Class::Comment + } else if first.is_ascii_alphabetic() || first == b'_' { + self.cursor += 1; + while bytes + .get(self.cursor) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + self.cursor += 1; + } + legacy_number_part = true; + Class::Word + } else if first.is_ascii_digit() { + while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) { + self.cursor += 1; + } + let mut valid = true; + if bytes.get(self.cursor) == Some(&b'.') { + self.cursor += 1; + let fraction = self.cursor; + while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) { + self.cursor += 1; + } + valid &= self.cursor != fraction; + } + if matches!(bytes.get(self.cursor), Some(b'e' | b'E')) { + self.cursor += 1; + if matches!(bytes.get(self.cursor), Some(b'+' | b'-')) { + self.cursor += 1; + } + let exponent = self.cursor; + while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) { + self.cursor += 1; + } + valid &= self.cursor != exponent; + } + if bytes + .get(self.cursor) + .is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.')) + { + valid = false; + while bytes + .get(self.cursor) + .is_some_and(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.')) + { + self.cursor += 1; + } + } + legacy_number_part = true; + if valid { Class::Number } else { Class::Error } + } else if first == b'"' { + self.cursor += 1; + let mut escaped = false; + let mut closed = false; + while self.cursor < bytes.len() { + let ch = source[self.cursor..] + .chars() + .next() + .expect("UTF-8 boundary"); + self.cursor += ch.len_utf8(); + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + closed = true; + break; + } + } + if closed && serde_json::from_str::(&source[start..self.cursor]).is_ok() { + Class::String + } else { + Class::Error + } + } else if b"{}();,[]:=@$+-*/<>!".contains(&first) { + self.cursor += 1; + if matches!(first, b'<' | b'>' | b'=' | b'!') && bytes.get(self.cursor) == Some(&b'=') { + self.cursor += 1; + } + legacy_number_part = matches!(first, b'+' | b'-'); + Class::Punctuation + } else { + self.cursor += source[start..] + .chars() + .next() + .expect("UTF-8 boundary") + .len_utf8(); + legacy_number_part = first == b'.'; + Class::Error + }; + Some(Token { + class, + span: Span { + start, + end: self.cursor, + }, + legacy_number_part, + }) + } +} + +/// Preserve the existing public token shapes, including greedy signed JSON +/// numbers and single-character rejection of newer expression operators. +pub(super) fn legacy(source: &str) -> Vec { + use super::{TokenClass, TokenSpan}; + let mut scanner = Scanner::new(source).peekable(); + let mut out = Vec::new(); + while let Some(token) = scanner.next() { + let mut span = token.span; + let first = source.as_bytes()[span.start]; + if first.is_ascii_digit() || matches!(first, b'+' | b'-' | b'.') { + while scanner + .peek() + .is_some_and(|next| next.legacy_number_part && next.span.start == span.end) + { + span.end = scanner.next().expect("peeked token").span.end; + } + let class = if serde_json::from_str::(&source[span.start..span.end]) + .is_ok() + { + TokenClass::Number + } else { + TokenClass::Error + }; + out.push(TokenSpan { class, span }); + continue; + } + let class = match token.class { + Class::Word => { + let word = &source[span.start..span.end]; + if super::reserved(word) || matches!(word, "true" | "false" | "none") { + TokenClass::Keyword + } else { + TokenClass::Identifier + } + } + Class::String => TokenClass::String, + Class::Whitespace => TokenClass::Whitespace, + Class::Comment => TokenClass::Comment, + Class::Punctuation => { + // The raw scanner joins comparison operators. Legacy syntax + // classified their ASCII characters individually. + for at in span.start..span.end { + let class = if b"{}();,[]:=@".contains(&source.as_bytes()[at]) { + TokenClass::Punctuation + } else { + TokenClass::Error + }; + out.push(TokenSpan { + class, + span: Span { + start: at, + end: at + 1, + }, + }); + } + continue; + } + Class::Error => TokenClass::Error, + Class::Number => unreachable!("number tokens start with a digit"), + }; + out.push(TokenSpan { class, span }); + } + out +} + +#[cfg(test)] +mod tests { + use super::super::{TokenClass, tokenize}; + + #[test] + fn legacy_adapter_retains_authored_numeric_operator_and_trivia_tokens() { + use TokenClass::*; + let cases: &[(&str, &[(TokenClass, &str)])] = &[ + ( + "-1.5 1e-2 -0", + &[ + (Number, "-1.5"), + (Whitespace, " "), + (Number, "1e-2"), + (Whitespace, " "), + (Number, "-0"), + ], + ), + ( + "1-2 1 -2 1 - 2", + &[ + (Error, "1-2"), + (Whitespace, " "), + (Number, "1"), + (Whitespace, " "), + (Number, "-2"), + (Whitespace, " "), + (Number, "1"), + (Whitespace, " "), + (Error, "-"), + (Whitespace, " "), + (Number, "2"), + ], + ), + ( + "+1 .5 1_0 0x1 1e 1e+", + &[ + (Error, "+1"), + (Whitespace, " "), + (Error, ".5"), + (Whitespace, " "), + (Error, "1_0"), + (Whitespace, " "), + (Error, "0x1"), + (Whitespace, " "), + (Error, "1e"), + (Whitespace, " "), + (Error, "1e+"), + ], + ), + ( + "<= != == >= / * $ !", + &[ + (Error, "<"), + (Punctuation, "="), + (Whitespace, " "), + (Error, "!"), + (Punctuation, "="), + (Whitespace, " "), + (Punctuation, "="), + (Punctuation, "="), + (Whitespace, " "), + (Error, ">"), + (Punctuation, "="), + (Whitespace, " "), + (Error, "/"), + (Whitespace, " "), + (Error, "*"), + (Whitespace, " "), + (Error, "$"), + (Whitespace, " "), + (Error, "!"), + ], + ), + ( + "// π\r\n\"雪\" @alias", + &[ + (Comment, "// π\r"), + (Whitespace, "\n"), + (String, "\"雪\""), + (Whitespace, " "), + (Punctuation, "@"), + (Identifier, "alias"), + ], + ), + ("\"incomplete 雪", &[(Error, "\"incomplete 雪")]), + ("\"bad\\q\"", &[(Error, "\"bad\\q\"")]), + ( + "雪\u{b}a", + &[(Error, "雪"), (Error, "\u{b}"), (Identifier, "a")], + ), + ( + "unit calculation quantity_kind if and List Number", + &[ + (Identifier, "unit"), + (Whitespace, " "), + (Identifier, "calculation"), + (Whitespace, " "), + (Identifier, "quantity_kind"), + (Whitespace, " "), + (Identifier, "if"), + (Whitespace, " "), + (Identifier, "and"), + (Whitespace, " "), + (Identifier, "List"), + (Whitespace, " "), + (Identifier, "Number"), + ], + ), + ]; + for (source, expected) in cases { + let actual = tokenize(source); + assert_eq!(actual.len(), expected.len(), "{source:?}"); + let mut offset = 0; + for (token, (class, spelling)) in actual.iter().zip(*expected) { + assert_eq!(token.class, *class, "{source:?} at {offset}"); + assert_eq!(token.span.start, offset); + offset += spelling.len(); + assert_eq!(token.span.end, offset); + assert_eq!(&source[token.span.start..token.span.end], *spelling); + } + assert_eq!(offset, source.len()); + } + } +}