feat(source): stage typed declaration syntax on shared raw tokens
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
#[path = "language/semantic_source.rs"]
|
||||
mod semantic_source;
|
||||
mod source3_bridge;
|
||||
mod raw_tokens;
|
||||
// Staged syntax foundation only. Public source8 admission waits for complete
|
||||
// declaration lowering, source updates and the root-owned resource gate.
|
||||
#[cfg(test)]
|
||||
@@ -152,86 +153,7 @@ pub struct Analysis {
|
||||
/// Tolerant highlighting tokens cover the whole input, including incomplete
|
||||
/// strings and unexpected characters. No model is created or modified.
|
||||
pub fn tokenize(source: &str) -> Vec<TokenSpan> {
|
||||
let mut tokens = Vec::new();
|
||||
let mut cursor = 0;
|
||||
while cursor < source.len() {
|
||||
let start = cursor;
|
||||
let ch = source[cursor..].chars().next().expect("in bounds");
|
||||
let class = if matches!(ch, ' ' | '\t' | '\r' | '\n') {
|
||||
cursor += 1;
|
||||
while source
|
||||
.as_bytes()
|
||||
.get(cursor)
|
||||
.is_some_and(|next| matches!(*next, b' ' | b'\t' | b'\r' | b'\n'))
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
TokenClass::Whitespace
|
||||
} else if source[cursor..].starts_with("//") {
|
||||
cursor += source[cursor..].find('\n').unwrap_or(source.len() - cursor);
|
||||
TokenClass::Comment
|
||||
} else if ch == '"' {
|
||||
cursor += 1;
|
||||
let mut escaped = false;
|
||||
let mut closed = false;
|
||||
while cursor < source.len() {
|
||||
let next = source[cursor..].chars().next().expect("in bounds");
|
||||
cursor += next.len_utf8();
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if next == '\\' {
|
||||
escaped = true;
|
||||
} else if next == '"' {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if closed && serde_json::from_str::<String>(&source[start..cursor]).is_ok() {
|
||||
TokenClass::String
|
||||
} else {
|
||||
TokenClass::Error
|
||||
}
|
||||
} else if ch.is_ascii_alphabetic() || ch == '_' {
|
||||
cursor += 1;
|
||||
while source
|
||||
.as_bytes()
|
||||
.get(cursor)
|
||||
.is_some_and(|next| next.is_ascii_alphanumeric() || *next == b'_')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
if reserved(&source[start..cursor])
|
||||
|| matches!(&source[start..cursor], "true" | "false" | "none")
|
||||
{
|
||||
TokenClass::Keyword
|
||||
} else {
|
||||
TokenClass::Identifier
|
||||
}
|
||||
} else if ch.is_ascii_digit() || matches!(ch, '-' | '+' | '.') {
|
||||
cursor += 1;
|
||||
while source.as_bytes().get(cursor).is_some_and(|next| {
|
||||
next.is_ascii_alphanumeric() || matches!(*next, b'.' | b'+' | b'-' | b'_')
|
||||
}) {
|
||||
cursor += 1;
|
||||
}
|
||||
if serde_json::from_str::<serde_json::Number>(&source[start..cursor]).is_ok() {
|
||||
TokenClass::Number
|
||||
} else {
|
||||
TokenClass::Error
|
||||
}
|
||||
} else if "{}();,[]:=@".contains(ch) {
|
||||
cursor += ch.len_utf8();
|
||||
TokenClass::Punctuation
|
||||
} else {
|
||||
cursor += ch.len_utf8();
|
||||
TokenClass::Error
|
||||
};
|
||||
tokens.push(TokenSpan {
|
||||
class,
|
||||
span: Span { start, end: cursor },
|
||||
});
|
||||
}
|
||||
tokens
|
||||
raw_tokens::legacy(source)
|
||||
}
|
||||
|
||||
pub fn lint(source: &str) -> Vec<Diagnostic> {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Unresolved typed source8 syntax, staged for the complete source adapter.
|
||||
//! This module does not admit a Project, resolve references or enable source8.
|
||||
mod budget;
|
||||
mod cursor;
|
||||
mod declarations;
|
||||
mod lex;
|
||||
mod syntax;
|
||||
|
||||
@@ -96,7 +98,10 @@ pub enum Binding {
|
||||
|
||||
pub fn parse(source: &str, budget: &mut Budget) -> Result<Binding, Error> {
|
||||
let tokens = lex::tokens(source, budget)?;
|
||||
syntax::parse(source, &tokens, budget)
|
||||
let mut cursor = cursor::Cursor::new(source, &tokens, budget);
|
||||
let binding = syntax::binding(&mut cursor)?;
|
||||
cursor.end()?;
|
||||
Ok(binding)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
//! A borrowed significant-token cursor. Spans always refer to the full source.
|
||||
use super::*;
|
||||
use lex::{Class, Token};
|
||||
|
||||
pub(super) struct Cursor<'a, 'b> {
|
||||
pub source: &'a str,
|
||||
pub tokens: &'a [Token],
|
||||
pub cursor: usize,
|
||||
pub frames: usize,
|
||||
pub budget: &'b mut Budget,
|
||||
}
|
||||
impl<'a, 'b> Cursor<'a, 'b> {
|
||||
pub(super) fn new(source: &'a str, tokens: &'a [Token], budget: &'b mut Budget) -> Self {
|
||||
Self {
|
||||
source,
|
||||
tokens,
|
||||
cursor: 0,
|
||||
frames: 0,
|
||||
budget,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Cursor<'_, '_> {
|
||||
pub(super) fn span(&self) -> Span {
|
||||
self.tokens.get(self.cursor).map_or(
|
||||
Span {
|
||||
start: self.source.len(),
|
||||
end: self.source.len(),
|
||||
},
|
||||
|token| token.span,
|
||||
)
|
||||
}
|
||||
pub(super) fn text(&self) -> &str {
|
||||
let span = self.span();
|
||||
&self.source[span.start..span.end]
|
||||
}
|
||||
pub(super) fn take(&mut self, expected: &str) -> Result<Span, Error> {
|
||||
if self.text() != expected {
|
||||
return Err(error(self.span(), format!("Expected '{expected}'.")));
|
||||
}
|
||||
let span = self.span();
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
pub(super) fn end(&self) -> Result<(), Error> {
|
||||
if self.cursor != self.tokens.len() {
|
||||
Err(error(
|
||||
self.span(),
|
||||
"Unexpected content after analysis expression.",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
pub(super) fn open(&mut self) -> Result<(), Error> {
|
||||
self.budget.frame(self.frames + 1, self.span())?;
|
||||
self.frames += 1;
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn close(&mut self) {
|
||||
self.frames -= 1;
|
||||
}
|
||||
pub(super) fn reference(&mut self) -> Result<Reference, Error> {
|
||||
self.budget.reference(self.span())?;
|
||||
self.name()
|
||||
}
|
||||
pub(super) fn name(&mut self) -> Result<Reference, Error> {
|
||||
let span = self.span();
|
||||
if !self
|
||||
.tokens
|
||||
.get(self.cursor)
|
||||
.is_some_and(|token| token.class == Class::Word)
|
||||
|| !crate::valid_alias(self.text())
|
||||
{
|
||||
return Err(error(span, "Expected a valid reference or parameter name."));
|
||||
}
|
||||
let name = self.text().to_owned();
|
||||
self.cursor += 1;
|
||||
Ok(Reference { name, span })
|
||||
}
|
||||
pub(super) fn quoted(&mut self) -> Result<String, Error> {
|
||||
let span = self.span();
|
||||
if !self
|
||||
.tokens
|
||||
.get(self.cursor)
|
||||
.is_some_and(|token| token.class == Class::String)
|
||||
{
|
||||
return Err(error(span, "Expected a quoted string."));
|
||||
}
|
||||
let value =
|
||||
serde_json::from_str(self.text()).map_err(|_| error(span, "Invalid quoted string."))?;
|
||||
self.cursor += 1;
|
||||
Ok(value)
|
||||
}
|
||||
pub(super) fn last_end(&self) -> usize {
|
||||
self.tokens[self.cursor - 1].span.end
|
||||
}
|
||||
pub(super) fn exact_literal(&mut self) -> Result<(Exact, LiteralSyntax), Error> {
|
||||
if self.text() == "ratio" {
|
||||
return self.ratio();
|
||||
}
|
||||
let span = self.span();
|
||||
if !self
|
||||
.tokens
|
||||
.get(self.cursor)
|
||||
.is_some_and(|token| token.class == Class::Number)
|
||||
{
|
||||
return Err(error(span, "Expected an exact numeric atom or ratio."));
|
||||
}
|
||||
let value = self
|
||||
.budget
|
||||
.exact(&self.source[span.start..span.end], span)?;
|
||||
self.cursor += 1;
|
||||
Ok((
|
||||
value,
|
||||
LiteralSyntax {
|
||||
span,
|
||||
tokens: vec![span],
|
||||
},
|
||||
))
|
||||
}
|
||||
pub(super) fn integer(&mut self) -> Result<Span, Error> {
|
||||
let span = self.span();
|
||||
let text = self.text();
|
||||
if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(error(span, "Expected unsigned integer digits."));
|
||||
}
|
||||
if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS {
|
||||
return Err(error(span, "Integer component exceeds digit limit."));
|
||||
}
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
pub(super) fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> {
|
||||
let first_token = self.cursor;
|
||||
let start = self.take("ratio")?.start;
|
||||
self.open()?;
|
||||
self.take("(")?;
|
||||
let negative = if self.text() == "-" {
|
||||
self.cursor += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let numerator = self.integer()?;
|
||||
self.take(",")?;
|
||||
let denominator = self.integer()?;
|
||||
let end = self.take(")")?.end;
|
||||
self.close();
|
||||
let n = &self.source[numerator.start..numerator.end];
|
||||
let d = &self.source[denominator.start..denominator.end];
|
||||
let length = 9 + usize::from(negative) + n.len() + d.len();
|
||||
if length > crate::analysis_numeric::MAX_LITERAL_BYTES {
|
||||
return Err(error(
|
||||
Span { start, end },
|
||||
"Numeric reconstruction exceeds literal byte limit.",
|
||||
));
|
||||
}
|
||||
let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" });
|
||||
Ok((
|
||||
self.budget.exact(&normalized, Span { start, end })?,
|
||||
LiteralSyntax {
|
||||
span: Span { start, end },
|
||||
tokens: self.tokens[first_token..self.cursor]
|
||||
.iter()
|
||||
.map(|token| token.span)
|
||||
.collect(),
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,629 @@
|
||||
//! Required typed declaration syntax. No model construction or resolution.
|
||||
use super::cursor::Cursor;
|
||||
use super::*;
|
||||
use crate::{Id, Layer};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DeclarationKind {
|
||||
QuantityKind,
|
||||
Unit,
|
||||
Calculation,
|
||||
Property(Layer),
|
||||
Check(Layer),
|
||||
Case(Layer),
|
||||
}
|
||||
|
||||
/// Contextual recognition only: never add these words to the global alias ban.
|
||||
pub fn declaration_kind(word: &str) -> Option<DeclarationKind> {
|
||||
use DeclarationKind::*;
|
||||
Some(match word {
|
||||
"quantity_kind" => QuantityKind,
|
||||
"unit" => Unit,
|
||||
"calculation" => Calculation,
|
||||
"oa_property" => Property(Layer::Oa),
|
||||
"sa_property" => Property(Layer::Sa),
|
||||
"oa_check" => Check(Layer::Oa),
|
||||
"sa_check" => Check(Layer::Sa),
|
||||
"oa_analysis" => Case(Layer::Oa),
|
||||
"sa_analysis" => Case(Layer::Sa),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct IdentitySyntax {
|
||||
/// Missing IDs stay unresolved here. The bundle stabilization owner assigns
|
||||
/// and inserts an ID once, then reparses the actual returned bytes.
|
||||
pub value: Option<Id>,
|
||||
pub keyword: Option<Span>,
|
||||
pub literal: Option<Span>,
|
||||
pub insertion: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Header {
|
||||
pub kind: DeclarationKind,
|
||||
pub kind_span: Span,
|
||||
pub alias: Reference,
|
||||
pub name: String,
|
||||
pub name_span: Span,
|
||||
pub identity: IdentitySyntax,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Field<T> {
|
||||
/// Complete field including its keyword and trailing semicolon.
|
||||
pub span: Span,
|
||||
pub keyword: Span,
|
||||
/// Same meaning as an ordinary top-level field value span.
|
||||
pub value_span: Span,
|
||||
pub value: T,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct ScalarSyntax {
|
||||
pub span: Span,
|
||||
pub kind: ScalarKind,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum ScalarKind {
|
||||
Boolean,
|
||||
Number,
|
||||
Quantity(Reference),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct TypeSyntax {
|
||||
pub span: Span,
|
||||
pub scalar: ScalarSyntax,
|
||||
pub list: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct ParameterSyntax {
|
||||
pub span: Span,
|
||||
pub name: Reference,
|
||||
pub identity: IdentitySyntax,
|
||||
pub value_type: TypeSyntax,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct DimensionEntry {
|
||||
pub axis: Span,
|
||||
pub exponent: Span,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct DimensionSyntax {
|
||||
pub exponents: [i16; 7],
|
||||
pub entries: Vec<DimensionEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum AlgebraSyntax {
|
||||
Linear,
|
||||
Affine(Reference),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct ExactSyntax {
|
||||
pub value: Exact,
|
||||
pub literal: LiteralSyntax,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct OverrideSyntax {
|
||||
pub span: Span,
|
||||
pub target: Reference,
|
||||
pub binding: Binding,
|
||||
pub binding_span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Body {
|
||||
QuantityKind {
|
||||
dimension: Field<DimensionSyntax>,
|
||||
algebra: Field<AlgebraSyntax>,
|
||||
},
|
||||
Unit {
|
||||
kind: Field<Reference>,
|
||||
scale: Field<ExactSyntax>,
|
||||
offset: Field<ExactSyntax>,
|
||||
symbol: Field<String>,
|
||||
},
|
||||
Calculation {
|
||||
parameters: Vec<ParameterSyntax>,
|
||||
result: Field<TypeSyntax>,
|
||||
body: Field<Binding>,
|
||||
},
|
||||
Property {
|
||||
subject: Field<Reference>,
|
||||
value_type: Field<TypeSyntax>,
|
||||
base: Field<Binding>,
|
||||
display_unit: Option<Field<Option<Reference>>>,
|
||||
},
|
||||
Check {
|
||||
subject: Option<Field<Option<Reference>>>,
|
||||
documented_constraint: Option<Field<Option<Reference>>>,
|
||||
predicate: Field<Binding>,
|
||||
},
|
||||
Case {
|
||||
subject: Option<Field<Option<Reference>>>,
|
||||
base: Field<()>,
|
||||
overrides: Field<Vec<OverrideSyntax>>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Declaration {
|
||||
pub header: Header,
|
||||
pub span: Span,
|
||||
pub body_span: Span,
|
||||
pub description: Option<Field<String>>,
|
||||
/// Analysis roots may explicitly spell `parent none`; definitions may not.
|
||||
pub parent: Option<Field<()>>,
|
||||
pub body: Body,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Fields {
|
||||
dimension: Option<Field<DimensionSyntax>>,
|
||||
algebra: Option<Field<AlgebraSyntax>>,
|
||||
kind: Option<Field<Reference>>,
|
||||
scale: Option<Field<ExactSyntax>>,
|
||||
offset: Option<Field<ExactSyntax>>,
|
||||
symbol: Option<Field<String>>,
|
||||
result: Option<Field<TypeSyntax>>,
|
||||
body: Option<Field<Binding>>,
|
||||
subject: Option<Field<Reference>>,
|
||||
optional_subject: Option<Field<Option<Reference>>>,
|
||||
value_type: Option<Field<TypeSyntax>>,
|
||||
base: Option<Field<Binding>>,
|
||||
display_unit: Option<Field<Option<Reference>>>,
|
||||
documented_constraint: Option<Field<Option<Reference>>>,
|
||||
predicate: Option<Field<Binding>>,
|
||||
baseline: Option<Field<()>>,
|
||||
overrides: Option<Field<Vec<OverrideSyntax>>>,
|
||||
}
|
||||
|
||||
fn required<T>(value: Option<T>, name: &str, at: Span) -> Result<T, Error> {
|
||||
value.ok_or_else(|| error(at, format!("Missing required field '{name}'.")))
|
||||
}
|
||||
|
||||
impl Fields {
|
||||
fn finish(
|
||||
self,
|
||||
kind: DeclarationKind,
|
||||
parameters: Vec<ParameterSyntax>,
|
||||
at: Span,
|
||||
) -> Result<Body, Error> {
|
||||
Ok(match kind {
|
||||
DeclarationKind::QuantityKind => Body::QuantityKind {
|
||||
dimension: required(self.dimension, "dimension", at)?,
|
||||
algebra: required(self.algebra, "algebra", at)?,
|
||||
},
|
||||
DeclarationKind::Unit => Body::Unit {
|
||||
kind: required(self.kind, "kind", at)?,
|
||||
scale: required(self.scale, "scale", at)?,
|
||||
offset: required(self.offset, "offset", at)?,
|
||||
symbol: required(self.symbol, "symbol", at)?,
|
||||
},
|
||||
DeclarationKind::Calculation => Body::Calculation {
|
||||
parameters,
|
||||
result: required(self.result, "result", at)?,
|
||||
body: required(self.body, "body", at)?,
|
||||
},
|
||||
DeclarationKind::Property(_) => Body::Property {
|
||||
subject: required(self.subject, "subject", at)?,
|
||||
value_type: required(self.value_type, "type", at)?,
|
||||
base: required(self.base, "base", at)?,
|
||||
display_unit: self.display_unit,
|
||||
},
|
||||
DeclarationKind::Check(_) => Body::Check {
|
||||
subject: self.optional_subject,
|
||||
documented_constraint: self.documented_constraint,
|
||||
predicate: required(self.predicate, "predicate", at)?,
|
||||
},
|
||||
DeclarationKind::Case(_) => Body::Case {
|
||||
subject: self.optional_subject,
|
||||
base: required(self.baseline, "base", at)?,
|
||||
overrides: required(self.overrides, "overrides", at)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume exactly one declaration from already charged full-file tokens.
|
||||
/// `version` is inherited from the caller's entry policy; fragments have no
|
||||
/// independent version header. Public parsing does not call this staged API.
|
||||
pub(super) fn parse(
|
||||
p: &mut Cursor<'_, '_>,
|
||||
version: u32,
|
||||
lexical_owner: Option<Id>,
|
||||
) -> Result<Declaration, Error> {
|
||||
p.budget.active()?;
|
||||
let kind_span = p.span();
|
||||
let kind = declaration_kind(p.text())
|
||||
.ok_or_else(|| error(kind_span, "Expected a short typed declaration word."))?;
|
||||
if version != 8 {
|
||||
return Err(error(kind_span, "Analysis declarations require source8."));
|
||||
}
|
||||
if lexical_owner.is_some() {
|
||||
return Err(error(
|
||||
kind_span,
|
||||
"Analysis declarations and definitions must be roots.",
|
||||
));
|
||||
}
|
||||
p.budget.entry(kind_span)?;
|
||||
p.cursor += 1;
|
||||
let alias = p.name()?;
|
||||
let name_span = p.span();
|
||||
let name = p.quoted()?;
|
||||
let identity = p.identity()?;
|
||||
let opening = p.take("{")?;
|
||||
let header = Header {
|
||||
kind,
|
||||
kind_span,
|
||||
alias,
|
||||
name,
|
||||
name_span,
|
||||
identity,
|
||||
span: Span {
|
||||
start: kind_span.start,
|
||||
end: opening.start,
|
||||
},
|
||||
};
|
||||
let mut fields = Fields::default();
|
||||
let mut description = None;
|
||||
let mut parent = None;
|
||||
let mut parameters = Vec::new();
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut parameter_names = BTreeSet::new();
|
||||
let mut parameter_ids = BTreeSet::new();
|
||||
while p.text() != "}" {
|
||||
let keyword = p.span();
|
||||
p.budget.entry(keyword)?;
|
||||
if p.text() == "input" && kind == DeclarationKind::Calculation {
|
||||
let parameter = p.parameter()?;
|
||||
if !parameter_names.insert(parameter.name.name.clone()) {
|
||||
return Err(error(parameter.name.span, "Duplicate parameter name."));
|
||||
}
|
||||
if let Some(id) = parameter.identity.value
|
||||
&& !parameter_ids.insert(id)
|
||||
{
|
||||
return Err(error(
|
||||
parameter.identity.literal.expect("explicit identity"),
|
||||
"Duplicate parameter identity.",
|
||||
));
|
||||
}
|
||||
parameters.push(parameter);
|
||||
continue;
|
||||
}
|
||||
// Borrow the original spelling for duplicate detection; no field-name
|
||||
// allocation and no map insertion can overwrite an earlier value.
|
||||
let source = p.source;
|
||||
let key = &source[keyword.start..keyword.end];
|
||||
if !seen.insert(key) {
|
||||
return Err(error(keyword, format!("Duplicate field '{key}'.")));
|
||||
}
|
||||
p.cursor += 1;
|
||||
use DeclarationKind::*;
|
||||
match (kind, key) {
|
||||
(_, "description") => description = Some(p.field(keyword, |p| p.quoted())?),
|
||||
(Property(_) | Check(_) | Case(_), "parent") => {
|
||||
parent = Some(p.field(keyword, |p| {
|
||||
p.take("none")?;
|
||||
Ok(())
|
||||
})?)
|
||||
}
|
||||
(QuantityKind, "dimension") => {
|
||||
fields.dimension = Some(p.field(keyword, |p| p.dimension())?)
|
||||
}
|
||||
(QuantityKind, "algebra") => fields.algebra = Some(p.field(keyword, |p| p.algebra())?),
|
||||
(Unit, "kind") => fields.kind = Some(p.field(keyword, |p| p.at_reference())?),
|
||||
(Unit, "scale") => fields.scale = Some(p.field(keyword, |p| p.exact_value())?),
|
||||
(Unit, "offset") => fields.offset = Some(p.field(keyword, |p| p.exact_value())?),
|
||||
(Unit, "symbol") => fields.symbol = Some(p.field(keyword, |p| p.quoted())?),
|
||||
(Calculation, "result") => fields.result = Some(p.field(keyword, |p| p.value_type())?),
|
||||
(Calculation, "body") => fields.body = Some(p.field(keyword, syntax::binding)?),
|
||||
(Property(_), "subject") => {
|
||||
fields.subject = Some(p.field(keyword, |p| p.at_reference())?)
|
||||
}
|
||||
(Property(_), "type") => {
|
||||
fields.value_type = Some(p.field(keyword, |p| p.value_type())?)
|
||||
}
|
||||
(Property(_), "base") => fields.base = Some(p.field(keyword, syntax::binding)?),
|
||||
(Property(_), "display_unit") => {
|
||||
fields.display_unit = Some(p.field(keyword, |p| p.optional_reference())?)
|
||||
}
|
||||
(Check(_) | Case(_), "subject") => {
|
||||
fields.optional_subject = Some(p.field(keyword, |p| p.optional_reference())?)
|
||||
}
|
||||
(Check(_), "documented_constraint") => {
|
||||
fields.documented_constraint = Some(p.field(keyword, |p| p.optional_reference())?)
|
||||
}
|
||||
(Check(_), "predicate") => fields.predicate = Some(p.field(keyword, syntax::binding)?),
|
||||
(Case(_), "base") => {
|
||||
fields.baseline = Some(p.field(keyword, |p| {
|
||||
p.take("ModelBaseline")?;
|
||||
Ok(())
|
||||
})?)
|
||||
}
|
||||
(Case(_), "overrides") => fields.overrides = Some(p.field(keyword, |p| p.overrides())?),
|
||||
_ => {
|
||||
return Err(error(
|
||||
keyword,
|
||||
format!("Unknown field '{key}' for this typed declaration."),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let closing = p.take("}")?;
|
||||
let body = fields.finish(kind, parameters, closing)?;
|
||||
Ok(Declaration {
|
||||
header,
|
||||
span: Span {
|
||||
start: kind_span.start,
|
||||
end: closing.end,
|
||||
},
|
||||
body_span: Span {
|
||||
start: opening.end,
|
||||
end: closing.start,
|
||||
},
|
||||
description,
|
||||
parent,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
impl Cursor<'_, '_> {
|
||||
fn field<T>(
|
||||
&mut self,
|
||||
keyword: Span,
|
||||
parse: impl FnOnce(&mut Self) -> Result<T, Error>,
|
||||
) -> Result<Field<T>, Error> {
|
||||
let start = self.span().start;
|
||||
let value = parse(self)?;
|
||||
let end = self.last_end();
|
||||
let terminator = self.take(";")?;
|
||||
Ok(Field {
|
||||
span: Span {
|
||||
start: keyword.start,
|
||||
end: terminator.end,
|
||||
},
|
||||
keyword,
|
||||
value_span: Span { start, end },
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
fn identity(&mut self) -> Result<IdentitySyntax, Error> {
|
||||
if self.text() != "id" {
|
||||
return Ok(IdentitySyntax {
|
||||
value: None,
|
||||
keyword: None,
|
||||
literal: None,
|
||||
insertion: Some(self.span().start),
|
||||
});
|
||||
}
|
||||
let keyword = self.take("id")?;
|
||||
let literal = self.span();
|
||||
let spelling = self.quoted()?;
|
||||
if spelling.len() != 36
|
||||
|| ![8, 13, 18, 23]
|
||||
.into_iter()
|
||||
.all(|at| spelling.as_bytes()[at] == b'-')
|
||||
{
|
||||
return Err(error(
|
||||
literal,
|
||||
"Expected UUID in 8-4-4-4-12 hexadecimal form.",
|
||||
));
|
||||
}
|
||||
let id = Id::parse_str(&spelling).map_err(|_| error(literal, "Invalid UUID identity."))?;
|
||||
Ok(IdentitySyntax {
|
||||
value: Some(id),
|
||||
keyword: Some(keyword),
|
||||
literal: Some(literal),
|
||||
insertion: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn parameter(&mut self) -> Result<ParameterSyntax, Error> {
|
||||
let start = self.take("input")?.start;
|
||||
let name = self.name()?;
|
||||
let identity = self.identity()?;
|
||||
self.take(":")?;
|
||||
let value_type = self.value_type()?;
|
||||
let end = self.take(";")?.end;
|
||||
Ok(ParameterSyntax {
|
||||
span: Span { start, end },
|
||||
name,
|
||||
identity,
|
||||
value_type,
|
||||
})
|
||||
}
|
||||
|
||||
fn at_reference(&mut self) -> Result<Reference, Error> {
|
||||
self.take("@")?;
|
||||
self.reference()
|
||||
}
|
||||
|
||||
fn optional_reference(&mut self) -> Result<Option<Reference>, Error> {
|
||||
if self.text() == "none" {
|
||||
self.cursor += 1;
|
||||
Ok(None)
|
||||
} else {
|
||||
self.at_reference().map(Some)
|
||||
}
|
||||
}
|
||||
|
||||
fn exact_value(&mut self) -> Result<ExactSyntax, Error> {
|
||||
let (value, literal) = self.exact_literal()?;
|
||||
Ok(ExactSyntax { value, literal })
|
||||
}
|
||||
|
||||
fn value_type(&mut self) -> Result<TypeSyntax, Error> {
|
||||
let start = self.span().start;
|
||||
self.budget.entry(self.span())?;
|
||||
let list = self.text() == "List";
|
||||
if list {
|
||||
self.open()?;
|
||||
self.cursor += 1;
|
||||
self.take("(")?;
|
||||
}
|
||||
let scalar = self.scalar_type()?;
|
||||
if list {
|
||||
self.take(")")?;
|
||||
self.close();
|
||||
}
|
||||
Ok(TypeSyntax {
|
||||
span: Span {
|
||||
start,
|
||||
end: self.last_end(),
|
||||
},
|
||||
scalar,
|
||||
list,
|
||||
})
|
||||
}
|
||||
|
||||
fn scalar_type(&mut self) -> Result<ScalarSyntax, Error> {
|
||||
let start = self.span().start;
|
||||
let kind = match self.text() {
|
||||
"Boolean" => {
|
||||
self.cursor += 1;
|
||||
ScalarKind::Boolean
|
||||
}
|
||||
"Number" => {
|
||||
self.cursor += 1;
|
||||
ScalarKind::Number
|
||||
}
|
||||
"Quantity" => {
|
||||
self.open()?;
|
||||
self.cursor += 1;
|
||||
self.take("(")?;
|
||||
let reference = self.at_reference()?;
|
||||
self.take(")")?;
|
||||
self.close();
|
||||
ScalarKind::Quantity(reference)
|
||||
}
|
||||
_ => {
|
||||
return Err(error(
|
||||
self.span(),
|
||||
"Expected Boolean, Number or Quantity(@kind); nested List is not a scalar type.",
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(ScalarSyntax {
|
||||
span: Span {
|
||||
start,
|
||||
end: self.last_end(),
|
||||
},
|
||||
kind,
|
||||
})
|
||||
}
|
||||
|
||||
fn dimension(&mut self) -> Result<DimensionSyntax, Error> {
|
||||
self.take("{")?;
|
||||
let mut exponents = [0; 7];
|
||||
let mut seen = [false; 7];
|
||||
let mut entries = Vec::new();
|
||||
while self.text() != "}" {
|
||||
let axis = self.span();
|
||||
self.budget.entry(axis)?;
|
||||
let index = match self.text() {
|
||||
"length" => 0,
|
||||
"mass" => 1,
|
||||
"time" => 2,
|
||||
"electric_current" => 3,
|
||||
"temperature" => 4,
|
||||
"amount" => 5,
|
||||
"luminous_intensity" => 6,
|
||||
_ => return Err(error(axis, "Unknown dimension axis.")),
|
||||
};
|
||||
if seen[index] {
|
||||
return Err(error(axis, "Duplicate dimension axis."));
|
||||
}
|
||||
seen[index] = true;
|
||||
self.cursor += 1;
|
||||
let start = self.span().start;
|
||||
let negative = self.text() == "-";
|
||||
if negative {
|
||||
self.cursor += 1;
|
||||
}
|
||||
let digits = self.integer()?;
|
||||
let value = self.source[digits.start..digits.end]
|
||||
.parse::<i16>()
|
||||
.map_err(|_| error(digits, "Dimension exponent exceeds -32..32."))?;
|
||||
if value > 32 {
|
||||
return Err(error(digits, "Dimension exponent exceeds -32..32."));
|
||||
}
|
||||
exponents[index] = if negative { -value } else { value };
|
||||
let exponent = Span {
|
||||
start,
|
||||
end: digits.end,
|
||||
};
|
||||
let end = self.take(";")?.end;
|
||||
entries.push(DimensionEntry {
|
||||
axis,
|
||||
exponent,
|
||||
span: Span {
|
||||
start: axis.start,
|
||||
end,
|
||||
},
|
||||
});
|
||||
}
|
||||
self.take("}")?;
|
||||
Ok(DimensionSyntax { exponents, entries })
|
||||
}
|
||||
|
||||
fn algebra(&mut self) -> Result<AlgebraSyntax, Error> {
|
||||
if self.text() == "Linear" {
|
||||
self.cursor += 1;
|
||||
return Ok(AlgebraSyntax::Linear);
|
||||
}
|
||||
self.take("Affine")?;
|
||||
self.open()?;
|
||||
self.take("(")?;
|
||||
let difference = self.at_reference()?;
|
||||
self.take(")")?;
|
||||
self.close();
|
||||
Ok(AlgebraSyntax::Affine(difference))
|
||||
}
|
||||
|
||||
fn overrides(&mut self) -> Result<Vec<OverrideSyntax>, Error> {
|
||||
self.take("{")?;
|
||||
let mut entries = Vec::new();
|
||||
let mut names = BTreeSet::new();
|
||||
while self.text() != "}" {
|
||||
let start = self.span().start;
|
||||
self.budget.entry(self.span())?;
|
||||
let target = self.at_reference()?;
|
||||
if !names.insert(target.name.clone()) {
|
||||
return Err(error(target.span, "Duplicate override target."));
|
||||
}
|
||||
self.take("=")?;
|
||||
let binding_start = self.span().start;
|
||||
let binding = syntax::binding(self)?;
|
||||
let binding_span = Span {
|
||||
start: binding_start,
|
||||
end: self.last_end(),
|
||||
};
|
||||
let end = self.take(";")?.end;
|
||||
entries.push(OverrideSyntax {
|
||||
span: Span { start, end },
|
||||
target,
|
||||
binding,
|
||||
binding_span,
|
||||
});
|
||||
}
|
||||
self.take("}")?;
|
||||
Ok(entries)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,531 @@
|
||||
use super::*;
|
||||
use crate::language::ParseLimits;
|
||||
use crate::language::analysis_source::budget::Limits;
|
||||
|
||||
fn limits() -> Limits {
|
||||
Limits::source8(ParseLimits::UNLIMITED)
|
||||
}
|
||||
fn budget() -> Budget {
|
||||
Budget::new(limits()).unwrap()
|
||||
}
|
||||
|
||||
fn declaration(source: &str) -> Declaration {
|
||||
let mut budget = budget();
|
||||
let tokens = lex::tokens(source, &mut budget).unwrap();
|
||||
let mut cursor = Cursor::new(source, &tokens, &mut budget);
|
||||
let result = parse(&mut cursor, 8, None).unwrap();
|
||||
cursor.end().unwrap();
|
||||
result
|
||||
}
|
||||
|
||||
fn failure(source: &str) -> Error {
|
||||
let mut budget = budget();
|
||||
let tokens = lex::tokens(source, &mut budget).unwrap();
|
||||
parse(&mut Cursor::new(source, &tokens, &mut budget), 8, None).unwrap_err()
|
||||
}
|
||||
|
||||
fn text(source: &str, span: Span) -> &str {
|
||||
&source[span.start..span.end]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_declarations_have_independent_typed_body_and_header_oracles() {
|
||||
let source = "quantity_kind duration \"Duration\" id \"188b911c-5f8b-431c-9eea-2d41f3953767\" { dimension { time 1; length -2; }; algebra Affine(@difference); description \"雪\"; }";
|
||||
let actual = declaration(source);
|
||||
assert_eq!(actual.header.kind, DeclarationKind::QuantityKind);
|
||||
assert_eq!(actual.header.alias.name, "duration");
|
||||
assert_eq!(actual.header.name, "Duration");
|
||||
assert_eq!(
|
||||
actual.header.identity.value,
|
||||
Some(Id::parse_str("188b911c-5f8b-431c-9eea-2d41f3953767").unwrap())
|
||||
);
|
||||
assert_eq!(actual.description.as_ref().unwrap().value, "雪");
|
||||
assert!(actual.parent.is_none());
|
||||
let Body::QuantityKind { dimension, algebra } = actual.body else {
|
||||
panic!("quantity kind")
|
||||
};
|
||||
assert_eq!(dimension.value.exponents, [-2, 0, 1, 0, 0, 0, 0]);
|
||||
assert_eq!(text(source, dimension.keyword), "dimension");
|
||||
assert_eq!(text(source, dimension.value_span), "{ time 1; length -2; }");
|
||||
assert_eq!(text(source, dimension.value.entries[1].exponent), "-2");
|
||||
let AlgebraSyntax::Affine(difference) = algebra.value else {
|
||||
panic!("affine")
|
||||
};
|
||||
assert_eq!(difference.name, "difference");
|
||||
|
||||
let Body::Unit { kind, scale, offset, symbol } = declaration("unit millisecond \"Millisecond\" { kind @duration; scale ratio (1, // divisor\n 1000); offset 0; symbol \"ms\"; }").body else { panic!("unit") };
|
||||
assert_eq!(kind.value.name, "duration");
|
||||
assert_eq!(scale.value.value.canonical_literal(), "ratio(1, 1000)");
|
||||
assert_eq!(offset.value.value.canonical_literal(), "0");
|
||||
assert_eq!(symbol.value, "ms");
|
||||
|
||||
let Body::Calculation { parameters, result, body } = declaration("calculation combine \"Combine durations\" { input a id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Quantity(@duration); input b id \"7a47cc1e-8ae5-4551-b2d8-8a68108e0976\" : Quantity(@duration); result Quantity(@duration); body $a + $b; }").body else { panic!("calculation") };
|
||||
assert_eq!(
|
||||
parameters
|
||||
.iter()
|
||||
.map(|p| p.name.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["a", "b"]
|
||||
);
|
||||
assert!(!result.value.list);
|
||||
assert!(
|
||||
matches!(result.value.scalar.kind, ScalarKind::Quantity(ref r) if r.name == "duration")
|
||||
);
|
||||
let Binding::Expression(Expression {
|
||||
kind: ExpressionKind::Binary { op, left, right },
|
||||
..
|
||||
}) = body.value
|
||||
else {
|
||||
panic!("binary")
|
||||
};
|
||||
assert_eq!(op, BinaryOp::Add);
|
||||
assert!(matches!(left.kind, ExpressionKind::Parameter(ref r) if r.name == "a"));
|
||||
assert!(matches!(right.kind, ExpressionKind::Parameter(ref r) if r.name == "b"));
|
||||
|
||||
for (word, layer) in [("oa_property", Layer::Oa), ("sa_property", Layer::Sa)] {
|
||||
let actual = declaration(&format!(
|
||||
"{word} receive_time \"Receive time\" {{ subject @receive; type Quantity(@duration); base 750 [@millisecond]; display_unit @millisecond; parent none; }}"
|
||||
));
|
||||
assert_eq!(actual.header.kind, DeclarationKind::Property(layer));
|
||||
assert!(actual.parent.is_some());
|
||||
let Body::Property {
|
||||
subject,
|
||||
value_type,
|
||||
base,
|
||||
display_unit,
|
||||
} = actual.body
|
||||
else {
|
||||
panic!("property")
|
||||
};
|
||||
assert_eq!(subject.value.name, "receive");
|
||||
assert!(matches!(
|
||||
value_type.value.scalar.kind,
|
||||
ScalarKind::Quantity(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
base.value,
|
||||
Binding::Expression(Expression {
|
||||
kind: ExpressionKind::Quantity { .. },
|
||||
..
|
||||
})
|
||||
));
|
||||
assert_eq!(display_unit.unwrap().value.unwrap().name, "millisecond");
|
||||
}
|
||||
for word in ["oa_check", "sa_check"] {
|
||||
let Body::Check { subject, documented_constraint, predicate } = declaration(&format!("{word} budget \"Response budget\" {{ subject none; documented_constraint @constraint; predicate @total_time <= @limit; }}")).body else { panic!("check") };
|
||||
assert!(subject.unwrap().value.is_none());
|
||||
assert_eq!(
|
||||
documented_constraint.unwrap().value.unwrap().name,
|
||||
"constraint"
|
||||
);
|
||||
assert!(matches!(
|
||||
predicate.value,
|
||||
Binding::Expression(Expression {
|
||||
kind: ExpressionKind::Binary {
|
||||
op: BinaryOp::LessOrEqual,
|
||||
..
|
||||
},
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
for word in ["oa_analysis", "sa_analysis"] {
|
||||
let Body::Case { subject, base, overrides } = declaration(&format!("{word} degraded \"Degraded radio\" {{ subject @respond; base ModelBaseline; overrides {{ @receive_time = unbound; }}; }}")).body else { panic!("case") };
|
||||
assert_eq!(subject.unwrap().value.unwrap().name, "respond");
|
||||
assert!(base.value_span.end > base.value_span.start);
|
||||
assert_eq!(overrides.value.len(), 1);
|
||||
assert_eq!(overrides.value[0].target.name, "receive_time");
|
||||
assert!(matches!(overrides.value[0].binding, Binding::Unbound(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_unbound_and_all_scalar_list_types_are_explicit_valid_syntax() {
|
||||
for (spelling, list, kind) in [
|
||||
("Boolean", false, 0),
|
||||
("Number", false, 1),
|
||||
("Quantity(@duration)", false, 2),
|
||||
("List(Boolean)", true, 0),
|
||||
("List(Number)", true, 1),
|
||||
("List(Quantity(@duration))", true, 2),
|
||||
] {
|
||||
let Body::Calculation {
|
||||
parameters,
|
||||
result,
|
||||
body,
|
||||
} = declaration(&format!(
|
||||
"calculation c \"C\" {{ result {spelling}; body unbound; }}"
|
||||
))
|
||||
.body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert!(parameters.is_empty());
|
||||
assert_eq!(result.value.list, list);
|
||||
assert_eq!(
|
||||
match result.value.scalar.kind {
|
||||
ScalarKind::Boolean => 0,
|
||||
ScalarKind::Number => 1,
|
||||
ScalarKind::Quantity(_) => 2,
|
||||
},
|
||||
kind
|
||||
);
|
||||
assert!(matches!(body.value, Binding::Unbound(_)));
|
||||
}
|
||||
let Body::QuantityKind { dimension, algebra } =
|
||||
declaration("quantity_kind q \"Q\" { dimension {}; algebra Linear; }").body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(dimension.value.exponents, [0; 7]);
|
||||
assert!(dimension.value.entries.is_empty());
|
||||
assert_eq!(algebra.value, AlgebraSyntax::Linear);
|
||||
let Body::Case {
|
||||
subject, overrides, ..
|
||||
} = declaration("oa_analysis c \"C\" { base ModelBaseline; overrides {}; }").body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert!(subject.is_none());
|
||||
assert!(overrides.value.is_empty());
|
||||
let Body::Check {
|
||||
subject,
|
||||
documented_constraint,
|
||||
..
|
||||
} = declaration("oa_check c \"C\" { predicate unbound; }").body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert!(subject.is_none() && documented_constraint.is_none());
|
||||
let Body::Property { display_unit, .. } =
|
||||
declaration("oa_property p \"P\" { subject @s; type Number; base unbound; }").body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert!(display_unit.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_required_fields_and_unknown_or_duplicate_entries_reject_locally() {
|
||||
let required_sets: &[(&str, &[&str])] = &[
|
||||
(
|
||||
"quantity_kind q \"Q\"",
|
||||
&["dimension {};", "algebra Linear;"],
|
||||
),
|
||||
(
|
||||
"unit u \"U\"",
|
||||
&["kind @q;", "scale 1;", "offset 0;", "symbol \"u\";"],
|
||||
),
|
||||
("calculation c \"C\"", &["result Number;", "body unbound;"]),
|
||||
(
|
||||
"oa_property p \"P\"",
|
||||
&["subject @s;", "type Number;", "base unbound;"],
|
||||
),
|
||||
("oa_check c \"C\"", &["predicate unbound;"]),
|
||||
(
|
||||
"oa_analysis a \"A\"",
|
||||
&["base ModelBaseline;", "overrides {};"],
|
||||
),
|
||||
];
|
||||
for (header, fields) in required_sets {
|
||||
declaration(&format!("{header} {{ {} }}", fields.join(" ")));
|
||||
for removed in 0..fields.len() {
|
||||
let remaining = fields
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, field)| (i != removed).then_some(*field))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let error = failure(&format!("{header} {{ {remaining} }}"));
|
||||
assert!(
|
||||
error.message.contains("Missing required field"),
|
||||
"{error:?}"
|
||||
);
|
||||
let duplicate = format!("{header} {{ {} {} }}", fields.join(" "), fields[removed]);
|
||||
assert!(failure(&duplicate).message.contains("Duplicate field"));
|
||||
}
|
||||
}
|
||||
for source in [
|
||||
"quantity_kind q \"Q\" { dimension { time 1; time 2; }; algebra Linear; }",
|
||||
"quantity_kind q \"Q\" { dimension { unknown 1; }; algebra Linear; }",
|
||||
"quantity_kind q \"Q\" { dimension { time 33; }; algebra Linear; }",
|
||||
"unit u \"U\" { kind @q; scale 1 + 2; offset 0; symbol \"u\"; }",
|
||||
"unit u \"U\" { kind @q; scale 1 [@u]; offset 0; symbol \"u\"; }",
|
||||
"unit u \"U\" { kind @q; scale ratio(1, 0); offset 0; symbol \"u\"; }",
|
||||
"calculation c \"C\" { result List(List(Number)); body unbound; }",
|
||||
"calculation c \"C\" { input x : Number; input x : Number; result Number; body unbound; }",
|
||||
"calculation c \"C\" { input x id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Number; input y id \"28ff40b0-1b8f-4735-8536-5a6ab21b56cb\" : Number; result Number; body unbound; }",
|
||||
"oa_property p \"P\" { subject none; type Number; base unbound; }",
|
||||
"oa_property p \"P\" { subject @s; type Number; base none; }",
|
||||
"oa_property p \"P\" { subject @s; type Number; base unbound; parent @s; }",
|
||||
"oa_analysis a \"A\" { base @another; overrides {}; }",
|
||||
"oa_analysis a \"A\" { base ModelBaseline; overrides { @p = clear; }; }",
|
||||
"oa_analysis a \"A\" { base ModelBaseline; overrides { @p = 1; @p = 2; }; }",
|
||||
"oa_check c \"C\" { predicate @f(x: 1, x: 2); }",
|
||||
"quantity_kind q \"Q\" { dimension {}; algebra Linear; parent none; }",
|
||||
"oa_check c \"C\" { predicate unbound; invented none; }",
|
||||
] {
|
||||
failure(source);
|
||||
}
|
||||
let duplicate = "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = 1; @p = 2; }; }";
|
||||
let error = failure(duplicate);
|
||||
assert_eq!(text(duplicate, error.span), "p");
|
||||
assert!(error.span.start > duplicate.find("@p").unwrap() + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn borrowed_cursor_preserves_full_file_spans_and_does_not_rescan_fields() {
|
||||
let source = "// 雪\r\ncalculation c \"C\" { input a : Number; result Number; body @combine(a: ratio (1, // first\r\n 2), b: $a) + @combine(a: 2, b: $a); }\r\nunit u \"U\" { kind @q; scale 1; offset 0; symbol \"u\"; }";
|
||||
let mut shared = budget();
|
||||
let tokens = lex::tokens(source, &mut shared).unwrap();
|
||||
let lexical = shared.usage;
|
||||
let mut cursor = Cursor::new(source, &tokens, &mut shared);
|
||||
let first = parse(&mut cursor, 8, None).unwrap();
|
||||
assert_eq!(cursor.text(), "unit");
|
||||
let first_end = first.span.end;
|
||||
let second = parse(&mut cursor, 8, None).unwrap();
|
||||
cursor.end().unwrap();
|
||||
assert!(second.span.start > first_end);
|
||||
assert_eq!(text(source, first.header.kind_span), "calculation");
|
||||
assert_eq!(text(source, first.header.name_span), "\"C\"");
|
||||
assert_eq!(text(source, second.header.alias.span), "u");
|
||||
assert_eq!(
|
||||
text(source, first.body_span).trim().strip_prefix("input"),
|
||||
Some(
|
||||
" a : Number; result Number; body @combine(a: ratio (1, // first\r\n 2), b: $a) + @combine(a: 2, b: $a);"
|
||||
)
|
||||
);
|
||||
let Body::Calculation {
|
||||
parameters, body, ..
|
||||
} = first.body
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(text(source, parameters[0].span), "input a : Number;");
|
||||
assert_eq!(text(source, parameters[0].value_type.span), "Number");
|
||||
let Binding::Expression(Expression {
|
||||
kind: ExpressionKind::Binary { left, right, .. },
|
||||
..
|
||||
}) = body.value
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
let ExpressionKind::Call {
|
||||
arguments: left, ..
|
||||
} = left.kind
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
let ExpressionKind::Call {
|
||||
arguments: right, ..
|
||||
} = right.kind
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
let spans = [
|
||||
left[0].parameter.span,
|
||||
left[1].parameter.span,
|
||||
right[0].parameter.span,
|
||||
right[1].parameter.span,
|
||||
];
|
||||
assert_eq!(spans.map(|span| text(source, span)), ["a", "b", "a", "b"]);
|
||||
assert!(spans.windows(2).all(|pair| pair[0].end < pair[1].start));
|
||||
let literal = left[0].value.literal.as_ref().unwrap();
|
||||
assert_eq!(text(source, literal.span), "ratio (1, // first\r\n 2)");
|
||||
assert_eq!(
|
||||
literal
|
||||
.tokens
|
||||
.iter()
|
||||
.map(|span| text(source, *span))
|
||||
.collect::<Vec<_>>(),
|
||||
["ratio", "(", "1", ",", "2", ")"]
|
||||
);
|
||||
assert_eq!(shared.usage.bytes, source.len());
|
||||
assert_eq!(shared.usage.bytes, lexical.bytes);
|
||||
assert_eq!(shared.usage.tokens, lexical.tokens);
|
||||
assert_eq!(shared.usage.references, 9); // two calls, four keys, two $a, unit kind.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_id_metadata_is_stable_and_parameter_ids_are_owner_scoped() {
|
||||
let source =
|
||||
"calculation c \"C\" { input a // identity here\r\n : Number; result Number; body $a; }";
|
||||
let first = declaration(source);
|
||||
let second = declaration(source);
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(
|
||||
first.header.identity.insertion,
|
||||
Some(source.find('{').unwrap())
|
||||
);
|
||||
let Body::Calculation { parameters, .. } = first.body else {
|
||||
panic!()
|
||||
};
|
||||
let id = ¶meters[0].identity;
|
||||
assert!(id.value.is_none() && id.literal.is_none() && id.keyword.is_none());
|
||||
assert_eq!(id.insertion, Some(source.find(':').unwrap()));
|
||||
let explicit = "28ff40b0-1b8f-4735-8536-5a6ab21b56cb";
|
||||
// Syntax does not add ParameterIds to a global identity registry, and an
|
||||
// explicit parameter ID can equal its own calculation's global UUID.
|
||||
for name in ["one", "two"] {
|
||||
let source = format!(
|
||||
"calculation {name} \"C\" id \"{explicit}\" {{ input a id \"{explicit}\" : Number; result Number; body $a; }}"
|
||||
);
|
||||
let actual = declaration(&source);
|
||||
let Body::Calculation { parameters, .. } = actual.body else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(parameters[0].identity.value, actual.header.identity.value);
|
||||
assert!(parameters[0].identity.insertion.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inherited_version_root_and_short_word_contracts_do_not_enable_public8() {
|
||||
let source = "oa_check c \"C\" { predicate unbound; }";
|
||||
for version in [1, 2, 3, 4, 5, 6, 7, 9] {
|
||||
let mut budget = budget();
|
||||
let tokens = lex::tokens(source, &mut budget).unwrap();
|
||||
let error = parse(
|
||||
&mut Cursor::new(source, &tokens, &mut budget),
|
||||
version,
|
||||
None,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(text(source, error.span), "oa_check");
|
||||
assert!(error.message.contains("source8"));
|
||||
}
|
||||
let mut budget = budget();
|
||||
let tokens = lex::tokens(source, &mut budget).unwrap();
|
||||
assert!(
|
||||
parse(
|
||||
&mut Cursor::new(source, &tokens, &mut budget),
|
||||
8,
|
||||
Some(Id::nil())
|
||||
)
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("roots")
|
||||
);
|
||||
for word in [
|
||||
"oa_engineering_property",
|
||||
"sa_engineering_property",
|
||||
"oa_analysis_check",
|
||||
"sa_analysis_check",
|
||||
"oa_analysis_case",
|
||||
"sa_analysis_case",
|
||||
] {
|
||||
assert!(declaration_kind(word).is_none());
|
||||
}
|
||||
for alias in [
|
||||
"unit",
|
||||
"calculation",
|
||||
"quantity_kind",
|
||||
"oa_property",
|
||||
"if",
|
||||
"and",
|
||||
"List",
|
||||
] {
|
||||
let old = format!("arcadia 7; project {alias} \"P\" {{}}");
|
||||
assert!(crate::language::parse(&old).is_ok(), "{old}");
|
||||
}
|
||||
let contextual =
|
||||
declaration("calculation unit \"C\" { input and : Number; result Number; body $and; }");
|
||||
assert_eq!(contextual.header.alias.name, "unit");
|
||||
assert!(crate::language::parse("arcadia 8; project p \"P\" {}").is_err());
|
||||
for version in 3..=7 {
|
||||
assert!(
|
||||
crate::language::parse(&format!(
|
||||
"arcadia {version}; project p \"P\" {{ {source} }}"
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declaration_budgets_share_literals_types_references_and_failure_state() {
|
||||
let source = "unit u \"U\" { kind @q; scale ratio(1, 3); offset ratio(2, 7); symbol \"u\"; }";
|
||||
let mut measured = budget();
|
||||
let tokens = lex::tokens(source, &mut measured).unwrap();
|
||||
parse(&mut Cursor::new(source, &tokens, &mut measured), 8, None).unwrap();
|
||||
let numeric = measured.numeric_units();
|
||||
assert!(numeric > 0);
|
||||
// All declaration, field and type entries share the same preallocation
|
||||
// ceiling. Measure a deterministic valid counterpart before reducing it.
|
||||
for (maximum, succeeds) in [
|
||||
(measured.usage.entries, true),
|
||||
(measured.usage.entries - 1, false),
|
||||
] {
|
||||
let mut profile = limits();
|
||||
profile.entries = maximum;
|
||||
let mut shared = Budget::new(profile).unwrap();
|
||||
let tokens = lex::tokens(source, &mut shared).unwrap();
|
||||
let result = parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None);
|
||||
assert_eq!(result.is_ok(), succeeds);
|
||||
if !succeeds {
|
||||
assert!(result.unwrap_err().message.contains("entries"));
|
||||
}
|
||||
}
|
||||
for (maximum, succeeds) in [(numeric, true), (numeric - 1, false)] {
|
||||
let mut profile = limits();
|
||||
profile.numeric_units = maximum;
|
||||
let mut shared = Budget::new(profile).unwrap();
|
||||
let tokens = lex::tokens(source, &mut shared).unwrap();
|
||||
let result = parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None);
|
||||
assert_eq!(result.is_ok(), succeeds);
|
||||
}
|
||||
let first =
|
||||
"calculation c \"C\" { input x : Quantity(@q); result Quantity(@q); body unbound; }";
|
||||
let second = "oa_property p \"P\" { subject @s; type Quantity(@q); base 1 [@u]; }";
|
||||
let mut profile = limits();
|
||||
profile.references = 4;
|
||||
let mut shared = Budget::new(profile).unwrap();
|
||||
let tokens = lex::tokens(first, &mut shared).unwrap();
|
||||
parse(&mut Cursor::new(first, &tokens, &mut shared), 8, None).unwrap();
|
||||
assert_eq!(shared.usage.references, 2);
|
||||
let tokens = lex::tokens(second, &mut shared).unwrap();
|
||||
let error = parse(&mut Cursor::new(second, &tokens, &mut shared), 8, None).unwrap_err();
|
||||
assert_eq!(text(second, error.span), "u");
|
||||
assert_eq!(shared.usage.references, 4);
|
||||
assert_eq!(shared.active().unwrap_err(), error);
|
||||
let numeric_after_failure = shared.numeric_units();
|
||||
assert!(lex::tokens("1", &mut shared).is_err());
|
||||
assert_eq!(shared.numeric_units(), numeric_after_failure);
|
||||
let unbound_tokens = lex::tokens("unbound", &mut budget()).unwrap();
|
||||
assert_eq!(
|
||||
syntax::binding(&mut Cursor::new("unbound", &unbound_tokens, &mut shared)).unwrap_err(),
|
||||
error
|
||||
);
|
||||
assert!(
|
||||
declaration("oa_check fresh \"Fresh\" { predicate true; }")
|
||||
.parent
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typed_frame_budget_and_truncated_declaration_cleanup_have_nearby_successes() {
|
||||
let source = "calculation c \"C\" { result List(Quantity(@q)); body unbound; }";
|
||||
for (maximum, succeeds) in [(2, true), (1, false)] {
|
||||
let mut profile = limits();
|
||||
profile.frames = maximum;
|
||||
let mut shared = Budget::new(profile).unwrap();
|
||||
let tokens = lex::tokens(source, &mut shared).unwrap();
|
||||
assert_eq!(
|
||||
parse(&mut Cursor::new(source, &tokens, &mut shared), 8, None).is_ok(),
|
||||
succeeds
|
||||
);
|
||||
}
|
||||
let source = "oa_analysis a \"A\" { base ModelBaseline; overrides { @p = if(true, ratio(1, 2), @f(a: 1)); }; }";
|
||||
let tokens = lex::tokens(source, &mut budget()).unwrap();
|
||||
for token in tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(text(source, token.span), "{" | "(" | "," | ":" | "="))
|
||||
{
|
||||
let prefix = &source[..token.span.end];
|
||||
let mut shared = budget();
|
||||
let tokens = lex::tokens(prefix, &mut shared).unwrap();
|
||||
assert!(parse(&mut Cursor::new(prefix, &tokens, &mut shared), 8, None).is_err());
|
||||
declaration("oa_check fresh \"Fresh\" { predicate true; }");
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Raw numeric tokens: finite coordinate checks remain in the existing reader.
|
||||
//! Source8 significant-token adapter over the shared raw scanner.
|
||||
use super::super::raw_tokens::{Class as RawClass, Scanner};
|
||||
use super::{Budget, Error, Span, error};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -8,11 +9,13 @@ pub enum Class {
|
||||
String,
|
||||
Punctuation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Token {
|
||||
pub class: Class,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
pub fn tokens(source: &str, budget: &mut Budget) -> Result<Vec<Token>, Error> {
|
||||
budget.bytes(
|
||||
source.len(),
|
||||
@@ -21,118 +24,29 @@ pub fn tokens(source: &str, budget: &mut Budget) -> Result<Vec<Token>, Error> {
|
||||
end: source.len(),
|
||||
},
|
||||
)?;
|
||||
let bytes = source.as_bytes();
|
||||
let mut cursor = 0;
|
||||
let mut scanner = Scanner::new(source);
|
||||
let mut result = Vec::new();
|
||||
while cursor < bytes.len() {
|
||||
let start = cursor;
|
||||
// Reserve before scanning/storing a token, including skipped trivia.
|
||||
budget.token(Span { start, end: start })?;
|
||||
let class = if bytes[cursor].is_ascii_whitespace() {
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() {
|
||||
cursor += 1;
|
||||
}
|
||||
None
|
||||
} else if bytes[cursor..].starts_with(b"//") {
|
||||
while cursor < bytes.len() && bytes[cursor] != b'\n' {
|
||||
cursor += 1;
|
||||
}
|
||||
None
|
||||
} else if bytes[cursor].is_ascii_alphabetic() || bytes[cursor] == b'_' {
|
||||
cursor += 1;
|
||||
while cursor < bytes.len()
|
||||
&& (bytes[cursor].is_ascii_alphanumeric() || bytes[cursor] == b'_')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
Some(Class::Word)
|
||||
} else if bytes[cursor].is_ascii_digit() {
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if bytes.get(cursor) == Some(&b'.') {
|
||||
cursor += 1;
|
||||
let fraction = cursor;
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if cursor == fraction {
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Expected decimal fraction digits.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if matches!(bytes.get(cursor), Some(b'e' | b'E')) {
|
||||
cursor += 1;
|
||||
if matches!(bytes.get(cursor), Some(b'+' | b'-')) {
|
||||
cursor += 1;
|
||||
}
|
||||
let exponent = cursor;
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if cursor == exponent {
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Expected exponent digits.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if bytes
|
||||
.get(cursor)
|
||||
.is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.'))
|
||||
{
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Malformed numeric atom.",
|
||||
));
|
||||
}
|
||||
Some(Class::Number)
|
||||
} else if bytes[cursor] == b'"' {
|
||||
cursor += 1;
|
||||
let mut closed = false;
|
||||
while cursor < bytes.len() {
|
||||
match bytes[cursor] {
|
||||
b'\\' => {
|
||||
cursor += 1;
|
||||
if cursor < bytes.len() {
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
b'"' => {
|
||||
cursor += 1;
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
_ => cursor += 1,
|
||||
}
|
||||
}
|
||||
if !closed {
|
||||
return Err(error(Span { start, end: cursor }, "Unclosed string."));
|
||||
}
|
||||
// Quoted field decoding remains the typed declaration consumer's job.
|
||||
Some(Class::String)
|
||||
} else if b"{}();,[]:=@$+-*/<>!".contains(&bytes[cursor]) {
|
||||
cursor += 1;
|
||||
if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') && bytes.get(cursor) == Some(&b'=')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
Some(Class::Punctuation)
|
||||
} else {
|
||||
let end = start + source[start..].chars().next().unwrap().len_utf8();
|
||||
return Err(error(
|
||||
Span { start, end },
|
||||
"Unexpected expression character.",
|
||||
));
|
||||
let mut offset = 0;
|
||||
while offset < source.len() {
|
||||
// Charge trivia too, before the scanner advances or a token is stored.
|
||||
budget.token(Span {
|
||||
start: offset,
|
||||
end: offset,
|
||||
})?;
|
||||
let token = scanner.next().expect("source remains");
|
||||
offset = token.span.end;
|
||||
let class = match token.class {
|
||||
RawClass::Word => Class::Word,
|
||||
RawClass::Number => Class::Number,
|
||||
RawClass::String => Class::String,
|
||||
RawClass::Punctuation => Class::Punctuation,
|
||||
RawClass::Whitespace | RawClass::Comment => continue,
|
||||
RawClass::Error => return Err(error(token.span, "Invalid source token.")),
|
||||
};
|
||||
if let Some(class) = class {
|
||||
result.push(Token {
|
||||
class,
|
||||
span: Span { start, end: cursor },
|
||||
});
|
||||
}
|
||||
result.push(Token {
|
||||
class,
|
||||
span: token.span,
|
||||
});
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Iterative Pratt machine. Source nesting does not recurse on the native stack.
|
||||
use super::cursor::Cursor;
|
||||
use super::*;
|
||||
use lex::{Class, Token};
|
||||
use lex::Class;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
enum SequenceKind {
|
||||
@@ -27,25 +28,11 @@ enum Task {
|
||||
Power(usize),
|
||||
AsKind(usize, Reference),
|
||||
}
|
||||
struct Parser<'a, 'b> {
|
||||
source: &'a str,
|
||||
tokens: &'a [Token],
|
||||
cursor: usize,
|
||||
frames: usize,
|
||||
budget: &'b mut Budget,
|
||||
}
|
||||
pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Result<Binding, Error> {
|
||||
let mut p = Parser {
|
||||
source,
|
||||
tokens,
|
||||
cursor: 0,
|
||||
frames: 0,
|
||||
budget,
|
||||
};
|
||||
pub(super) fn binding(p: &mut Cursor<'_, '_>) -> Result<Binding, Error> {
|
||||
p.budget.active()?;
|
||||
if p.text() == "unbound" {
|
||||
let span = p.span();
|
||||
p.cursor += 1;
|
||||
p.end()?;
|
||||
return Ok(Binding::Unbound(span));
|
||||
}
|
||||
let mut tasks = vec![Task::Expression(0)];
|
||||
@@ -109,7 +96,7 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu
|
||||
);
|
||||
}
|
||||
"ratio" => {
|
||||
let (value, literal) = p.ratio()?;
|
||||
let (value, literal) = p.exact_literal()?;
|
||||
values.push(p.number(value, literal)?);
|
||||
}
|
||||
"abs" | "not" | "if" | "sum" => {
|
||||
@@ -146,16 +133,8 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu
|
||||
.get(p.cursor)
|
||||
.is_some_and(|token| token.class == Class::Number) =>
|
||||
{
|
||||
let span = p.span();
|
||||
let value = p.budget.exact(&p.source[span.start..span.end], span)?;
|
||||
p.cursor += 1;
|
||||
values.push(p.number(
|
||||
value,
|
||||
LiteralSyntax {
|
||||
span,
|
||||
tokens: vec![span],
|
||||
},
|
||||
)?);
|
||||
let (value, literal) = p.exact_literal()?;
|
||||
values.push(p.number(value, literal)?);
|
||||
}
|
||||
_ => return Err(error(p.span(), "Expected an analysis expression.")),
|
||||
}
|
||||
@@ -317,7 +296,6 @@ pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Resu
|
||||
}
|
||||
}
|
||||
}
|
||||
p.end()?;
|
||||
Ok(Binding::Expression(
|
||||
values.pop().expect("one complete expression"),
|
||||
))
|
||||
@@ -332,110 +310,7 @@ impl Sequence {
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Parser<'_, '_> {
|
||||
fn span(&self) -> Span {
|
||||
self.tokens.get(self.cursor).map_or(
|
||||
Span {
|
||||
start: self.source.len(),
|
||||
end: self.source.len(),
|
||||
},
|
||||
|token| token.span,
|
||||
)
|
||||
}
|
||||
fn text(&self) -> &str {
|
||||
let span = self.span();
|
||||
&self.source[span.start..span.end]
|
||||
}
|
||||
fn take(&mut self, expected: &str) -> Result<Span, Error> {
|
||||
if self.text() != expected {
|
||||
return Err(error(self.span(), format!("Expected '{expected}'.")));
|
||||
}
|
||||
let span = self.span();
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
fn end(&self) -> Result<(), Error> {
|
||||
if self.cursor != self.tokens.len() {
|
||||
Err(error(
|
||||
self.span(),
|
||||
"Unexpected content after analysis expression.",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn open(&mut self) -> Result<(), Error> {
|
||||
self.budget.frame(self.frames + 1, self.span())?;
|
||||
self.frames += 1;
|
||||
Ok(())
|
||||
}
|
||||
fn close(&mut self) {
|
||||
self.frames -= 1;
|
||||
}
|
||||
fn reference(&mut self) -> Result<Reference, Error> {
|
||||
let span = self.span();
|
||||
if !self
|
||||
.tokens
|
||||
.get(self.cursor)
|
||||
.is_some_and(|token| token.class == Class::Word)
|
||||
|| !crate::valid_alias(self.text())
|
||||
{
|
||||
return Err(error(span, "Expected a valid reference or parameter name."));
|
||||
}
|
||||
self.budget.reference(span)?;
|
||||
let name = self.text().to_owned();
|
||||
self.cursor += 1;
|
||||
Ok(Reference { name, span })
|
||||
}
|
||||
fn integer(&mut self) -> Result<Span, Error> {
|
||||
let span = self.span();
|
||||
let text = self.text();
|
||||
if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(error(span, "Expected unsigned integer digits."));
|
||||
}
|
||||
if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS {
|
||||
return Err(error(span, "Integer component exceeds digit limit."));
|
||||
}
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> {
|
||||
let first_token = self.cursor;
|
||||
let start = self.take("ratio")?.start;
|
||||
self.open()?;
|
||||
self.take("(")?;
|
||||
let negative = if self.text() == "-" {
|
||||
self.cursor += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let numerator = self.integer()?;
|
||||
self.take(",")?;
|
||||
let denominator = self.integer()?;
|
||||
let end = self.take(")")?.end;
|
||||
self.close();
|
||||
let n = &self.source[numerator.start..numerator.end];
|
||||
let d = &self.source[denominator.start..denominator.end];
|
||||
let length = 9 + usize::from(negative) + n.len() + d.len();
|
||||
if length > crate::analysis_numeric::MAX_LITERAL_BYTES {
|
||||
return Err(error(
|
||||
Span { start, end },
|
||||
"Numeric reconstruction exceeds literal byte limit.",
|
||||
));
|
||||
}
|
||||
let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" });
|
||||
Ok((
|
||||
self.budget.exact(&normalized, Span { start, end })?,
|
||||
LiteralSyntax {
|
||||
span: Span { start, end },
|
||||
tokens: self.tokens[first_token..self.cursor]
|
||||
.iter()
|
||||
.map(|token| token.span)
|
||||
.collect(),
|
||||
},
|
||||
))
|
||||
}
|
||||
impl Cursor<'_, '_> {
|
||||
fn number(&mut self, magnitude: Exact, literal: LiteralSyntax) -> Result<Expression, Error> {
|
||||
let Span { start, end } = literal.span;
|
||||
let mut expression = if self.text() == "[" {
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
//! One raw lexical scan for legacy adapters and staged typed source syntax.
|
||||
//! Numbers remain source spans; only consumers choose a numeric domain.
|
||||
use super::Span;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Class {
|
||||
Word,
|
||||
Number,
|
||||
String,
|
||||
Punctuation,
|
||||
Whitespace,
|
||||
Comment,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(super) struct Token {
|
||||
pub class: Class,
|
||||
pub span: Span,
|
||||
/// Entire token belongs to the old greedy numeric-token alphabet.
|
||||
pub legacy_number_part: bool,
|
||||
}
|
||||
|
||||
pub(super) struct Scanner<'a> {
|
||||
source: &'a str,
|
||||
cursor: usize,
|
||||
}
|
||||
|
||||
impl<'a> Scanner<'a> {
|
||||
pub fn new(source: &'a str) -> Self {
|
||||
Self { source, cursor: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for Scanner<'_> {
|
||||
type Item = Token;
|
||||
|
||||
fn next(&mut self) -> Option<Token> {
|
||||
let source = self.source;
|
||||
let bytes = source.as_bytes();
|
||||
let start = self.cursor;
|
||||
let first = *bytes.get(start)?;
|
||||
let mut legacy_number_part = false;
|
||||
let class = if matches!(first, b' ' | b'\t' | b'\r' | b'\n') {
|
||||
self.cursor += 1;
|
||||
while bytes
|
||||
.get(self.cursor)
|
||||
.is_some_and(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n'))
|
||||
{
|
||||
self.cursor += 1;
|
||||
}
|
||||
Class::Whitespace
|
||||
} else if bytes[start..].starts_with(b"//") {
|
||||
self.cursor += 2;
|
||||
while bytes.get(self.cursor).is_some_and(|b| *b != b'\n') {
|
||||
self.cursor += 1;
|
||||
}
|
||||
Class::Comment
|
||||
} else if first.is_ascii_alphabetic() || first == b'_' {
|
||||
self.cursor += 1;
|
||||
while bytes
|
||||
.get(self.cursor)
|
||||
.is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_')
|
||||
{
|
||||
self.cursor += 1;
|
||||
}
|
||||
legacy_number_part = true;
|
||||
Class::Word
|
||||
} else if first.is_ascii_digit() {
|
||||
while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) {
|
||||
self.cursor += 1;
|
||||
}
|
||||
let mut valid = true;
|
||||
if bytes.get(self.cursor) == Some(&b'.') {
|
||||
self.cursor += 1;
|
||||
let fraction = self.cursor;
|
||||
while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) {
|
||||
self.cursor += 1;
|
||||
}
|
||||
valid &= self.cursor != fraction;
|
||||
}
|
||||
if matches!(bytes.get(self.cursor), Some(b'e' | b'E')) {
|
||||
self.cursor += 1;
|
||||
if matches!(bytes.get(self.cursor), Some(b'+' | b'-')) {
|
||||
self.cursor += 1;
|
||||
}
|
||||
let exponent = self.cursor;
|
||||
while bytes.get(self.cursor).is_some_and(u8::is_ascii_digit) {
|
||||
self.cursor += 1;
|
||||
}
|
||||
valid &= self.cursor != exponent;
|
||||
}
|
||||
if bytes
|
||||
.get(self.cursor)
|
||||
.is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.'))
|
||||
{
|
||||
valid = false;
|
||||
while bytes
|
||||
.get(self.cursor)
|
||||
.is_some_and(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.'))
|
||||
{
|
||||
self.cursor += 1;
|
||||
}
|
||||
}
|
||||
legacy_number_part = true;
|
||||
if valid { Class::Number } else { Class::Error }
|
||||
} else if first == b'"' {
|
||||
self.cursor += 1;
|
||||
let mut escaped = false;
|
||||
let mut closed = false;
|
||||
while self.cursor < bytes.len() {
|
||||
let ch = source[self.cursor..]
|
||||
.chars()
|
||||
.next()
|
||||
.expect("UTF-8 boundary");
|
||||
self.cursor += ch.len_utf8();
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if ch == '\\' {
|
||||
escaped = true;
|
||||
} else if ch == '"' {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if closed && serde_json::from_str::<String>(&source[start..self.cursor]).is_ok() {
|
||||
Class::String
|
||||
} else {
|
||||
Class::Error
|
||||
}
|
||||
} else if b"{}();,[]:=@$+-*/<>!".contains(&first) {
|
||||
self.cursor += 1;
|
||||
if matches!(first, b'<' | b'>' | b'=' | b'!') && bytes.get(self.cursor) == Some(&b'=') {
|
||||
self.cursor += 1;
|
||||
}
|
||||
legacy_number_part = matches!(first, b'+' | b'-');
|
||||
Class::Punctuation
|
||||
} else {
|
||||
self.cursor += source[start..]
|
||||
.chars()
|
||||
.next()
|
||||
.expect("UTF-8 boundary")
|
||||
.len_utf8();
|
||||
legacy_number_part = first == b'.';
|
||||
Class::Error
|
||||
};
|
||||
Some(Token {
|
||||
class,
|
||||
span: Span {
|
||||
start,
|
||||
end: self.cursor,
|
||||
},
|
||||
legacy_number_part,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Preserve the existing public token shapes, including greedy signed JSON
|
||||
/// numbers and single-character rejection of newer expression operators.
|
||||
pub(super) fn legacy(source: &str) -> Vec<super::TokenSpan> {
|
||||
use super::{TokenClass, TokenSpan};
|
||||
let mut scanner = Scanner::new(source).peekable();
|
||||
let mut out = Vec::new();
|
||||
while let Some(token) = scanner.next() {
|
||||
let mut span = token.span;
|
||||
let first = source.as_bytes()[span.start];
|
||||
if first.is_ascii_digit() || matches!(first, b'+' | b'-' | b'.') {
|
||||
while scanner
|
||||
.peek()
|
||||
.is_some_and(|next| next.legacy_number_part && next.span.start == span.end)
|
||||
{
|
||||
span.end = scanner.next().expect("peeked token").span.end;
|
||||
}
|
||||
let class = if serde_json::from_str::<serde_json::Number>(&source[span.start..span.end])
|
||||
.is_ok()
|
||||
{
|
||||
TokenClass::Number
|
||||
} else {
|
||||
TokenClass::Error
|
||||
};
|
||||
out.push(TokenSpan { class, span });
|
||||
continue;
|
||||
}
|
||||
let class = match token.class {
|
||||
Class::Word => {
|
||||
let word = &source[span.start..span.end];
|
||||
if super::reserved(word) || matches!(word, "true" | "false" | "none") {
|
||||
TokenClass::Keyword
|
||||
} else {
|
||||
TokenClass::Identifier
|
||||
}
|
||||
}
|
||||
Class::String => TokenClass::String,
|
||||
Class::Whitespace => TokenClass::Whitespace,
|
||||
Class::Comment => TokenClass::Comment,
|
||||
Class::Punctuation => {
|
||||
// The raw scanner joins comparison operators. Legacy syntax
|
||||
// classified their ASCII characters individually.
|
||||
for at in span.start..span.end {
|
||||
let class = if b"{}();,[]:=@".contains(&source.as_bytes()[at]) {
|
||||
TokenClass::Punctuation
|
||||
} else {
|
||||
TokenClass::Error
|
||||
};
|
||||
out.push(TokenSpan {
|
||||
class,
|
||||
span: Span {
|
||||
start: at,
|
||||
end: at + 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Class::Error => TokenClass::Error,
|
||||
Class::Number => unreachable!("number tokens start with a digit"),
|
||||
};
|
||||
out.push(TokenSpan { class, span });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{TokenClass, tokenize};
|
||||
|
||||
#[test]
|
||||
fn legacy_adapter_retains_authored_numeric_operator_and_trivia_tokens() {
|
||||
use TokenClass::*;
|
||||
let cases: &[(&str, &[(TokenClass, &str)])] = &[
|
||||
(
|
||||
"-1.5 1e-2 -0",
|
||||
&[
|
||||
(Number, "-1.5"),
|
||||
(Whitespace, " "),
|
||||
(Number, "1e-2"),
|
||||
(Whitespace, " "),
|
||||
(Number, "-0"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"1-2 1 -2 1 - 2",
|
||||
&[
|
||||
(Error, "1-2"),
|
||||
(Whitespace, " "),
|
||||
(Number, "1"),
|
||||
(Whitespace, " "),
|
||||
(Number, "-2"),
|
||||
(Whitespace, " "),
|
||||
(Number, "1"),
|
||||
(Whitespace, " "),
|
||||
(Error, "-"),
|
||||
(Whitespace, " "),
|
||||
(Number, "2"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"+1 .5 1_0 0x1 1e 1e+",
|
||||
&[
|
||||
(Error, "+1"),
|
||||
(Whitespace, " "),
|
||||
(Error, ".5"),
|
||||
(Whitespace, " "),
|
||||
(Error, "1_0"),
|
||||
(Whitespace, " "),
|
||||
(Error, "0x1"),
|
||||
(Whitespace, " "),
|
||||
(Error, "1e"),
|
||||
(Whitespace, " "),
|
||||
(Error, "1e+"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"<= != == >= / * $ !",
|
||||
&[
|
||||
(Error, "<"),
|
||||
(Punctuation, "="),
|
||||
(Whitespace, " "),
|
||||
(Error, "!"),
|
||||
(Punctuation, "="),
|
||||
(Whitespace, " "),
|
||||
(Punctuation, "="),
|
||||
(Punctuation, "="),
|
||||
(Whitespace, " "),
|
||||
(Error, ">"),
|
||||
(Punctuation, "="),
|
||||
(Whitespace, " "),
|
||||
(Error, "/"),
|
||||
(Whitespace, " "),
|
||||
(Error, "*"),
|
||||
(Whitespace, " "),
|
||||
(Error, "$"),
|
||||
(Whitespace, " "),
|
||||
(Error, "!"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"// π\r\n\"雪\" @alias",
|
||||
&[
|
||||
(Comment, "// π\r"),
|
||||
(Whitespace, "\n"),
|
||||
(String, "\"雪\""),
|
||||
(Whitespace, " "),
|
||||
(Punctuation, "@"),
|
||||
(Identifier, "alias"),
|
||||
],
|
||||
),
|
||||
("\"incomplete 雪", &[(Error, "\"incomplete 雪")]),
|
||||
("\"bad\\q\"", &[(Error, "\"bad\\q\"")]),
|
||||
(
|
||||
"雪\u{b}a",
|
||||
&[(Error, "雪"), (Error, "\u{b}"), (Identifier, "a")],
|
||||
),
|
||||
(
|
||||
"unit calculation quantity_kind if and List Number",
|
||||
&[
|
||||
(Identifier, "unit"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "calculation"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "quantity_kind"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "if"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "and"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "List"),
|
||||
(Whitespace, " "),
|
||||
(Identifier, "Number"),
|
||||
],
|
||||
),
|
||||
];
|
||||
for (source, expected) in cases {
|
||||
let actual = tokenize(source);
|
||||
assert_eq!(actual.len(), expected.len(), "{source:?}");
|
||||
let mut offset = 0;
|
||||
for (token, (class, spelling)) in actual.iter().zip(*expected) {
|
||||
assert_eq!(token.class, *class, "{source:?} at {offset}");
|
||||
assert_eq!(token.span.start, offset);
|
||||
offset += spelling.len();
|
||||
assert_eq!(token.span.end, offset);
|
||||
assert_eq!(&source[token.span.start..token.span.end], *spelling);
|
||||
}
|
||||
assert_eq!(offset, source.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user