Stage bounded source8 expression syntax and raw token spans
This commit is contained in:
@@ -1,6 +1,10 @@
|
||||
//! Typed OA/SA language, tolerant source analysis, and explicit V1/V2/V3/V4 migration.
|
||||
#[path = "language/semantic_source.rs"]
|
||||
mod semantic_source;
|
||||
// Staged syntax foundation only. Public source8 admission waits for complete
|
||||
// declaration lowering, source updates and the root-owned resource gate.
|
||||
#[cfg(test)]
|
||||
mod analysis_source;
|
||||
#[path = "migration_v2/source.rs"]
|
||||
mod v2_source;
|
||||
use crate::*;
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
//! Unresolved typed source8 syntax, staged for the complete source adapter.
|
||||
//! This module does not admit a Project, resolve references or enable source8.
|
||||
mod budget;
|
||||
mod lex;
|
||||
mod syntax;
|
||||
|
||||
use super::Span;
|
||||
use crate::analysis::{BinaryOp, UnaryOp};
|
||||
use crate::analysis_numeric::Exact;
|
||||
use budget::Budget;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Error {
|
||||
pub span: Span,
|
||||
pub message: String,
|
||||
}
|
||||
fn error(span: Span, message: impl Into<String>) -> Error {
|
||||
Error {
|
||||
span,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Reference {
|
||||
pub name: String,
|
||||
pub span: Span,
|
||||
}
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Argument {
|
||||
pub parameter: Reference,
|
||||
pub value: Expression,
|
||||
}
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Expression {
|
||||
pub kind: Kind,
|
||||
pub span: Span,
|
||||
pub depth: usize,
|
||||
pub grouped: bool,
|
||||
/// Original numeric tokens, including constructor punctuation, excluding
|
||||
/// trivia. The full span retains intervening comments and spelling.
|
||||
pub literal: Option<LiteralSyntax>,
|
||||
}
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct LiteralSyntax {
|
||||
pub span: Span,
|
||||
pub tokens: Vec<Span>,
|
||||
}
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Boolean(bool),
|
||||
Number(Exact),
|
||||
Quantity {
|
||||
magnitude: Exact,
|
||||
unit: Reference,
|
||||
},
|
||||
Property(Reference),
|
||||
Parameter(Reference),
|
||||
List(Vec<Expression>),
|
||||
Unary {
|
||||
op: UnaryOp,
|
||||
operand: Box<Expression>,
|
||||
},
|
||||
Binary {
|
||||
op: BinaryOp,
|
||||
left: Box<Expression>,
|
||||
right: Box<Expression>,
|
||||
},
|
||||
If {
|
||||
condition: Box<Expression>,
|
||||
when_true: Box<Expression>,
|
||||
when_false: Box<Expression>,
|
||||
},
|
||||
Call {
|
||||
definition: Reference,
|
||||
arguments: Vec<Argument>,
|
||||
},
|
||||
Sum {
|
||||
values: Box<Expression>,
|
||||
initial: Box<Expression>,
|
||||
},
|
||||
Power {
|
||||
base: Box<Expression>,
|
||||
exponent: i8,
|
||||
},
|
||||
AsKind {
|
||||
kind: Reference,
|
||||
value: Box<Expression>,
|
||||
},
|
||||
}
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Binding {
|
||||
Unbound(Span),
|
||||
Expression(Expression),
|
||||
}
|
||||
|
||||
pub fn parse(source: &str, budget: &mut Budget) -> Result<Binding, Error> {
|
||||
let tokens = lex::tokens(source, budget)?;
|
||||
syntax::parse(source, &tokens, budget)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,137 @@
|
||||
//! Candidate limits for measured adoption; one owner spans all parsed files.
|
||||
use super::{Error, Span, error};
|
||||
use crate::analysis_numeric::{Exact, WorkLimits, WorkMeter};
|
||||
use crate::language::ParseLimits;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Limits {
|
||||
pub bytes: usize,
|
||||
pub tokens: usize,
|
||||
pub entries: usize,
|
||||
pub references: usize,
|
||||
pub frames: usize,
|
||||
pub numeric_units: u64,
|
||||
}
|
||||
impl Limits {
|
||||
pub fn source8(caller: ParseLimits) -> Self {
|
||||
let bytes = caller.max_source_bytes.min(64 * 1024 * 1024);
|
||||
Self {
|
||||
bytes,
|
||||
tokens: bytes.min(1_048_576),
|
||||
entries: caller.max_records.min(262_144),
|
||||
references: caller.max_step_references.min(500_000),
|
||||
frames: 256,
|
||||
numeric_units: 50_000_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Usage {
|
||||
pub bytes: usize,
|
||||
pub tokens: usize,
|
||||
pub entries: usize,
|
||||
pub references: usize,
|
||||
pub deepest_frame: usize,
|
||||
}
|
||||
pub struct Budget {
|
||||
limits: Limits,
|
||||
pub usage: Usage,
|
||||
numeric: WorkMeter,
|
||||
failed: Option<Error>,
|
||||
}
|
||||
impl Budget {
|
||||
pub fn new(limits: Limits) -> Result<Self, Error> {
|
||||
let hard = Limits::source8(ParseLimits::UNLIMITED);
|
||||
if limits.bytes > hard.bytes
|
||||
|| limits.tokens > hard.tokens
|
||||
|| limits.entries > hard.entries
|
||||
|| limits.references > hard.references
|
||||
|| limits.frames > hard.frames
|
||||
|| limits.numeric_units > hard.numeric_units
|
||||
{
|
||||
return Err(error(
|
||||
Span::default(),
|
||||
"Source8 profile cannot exceed candidate hard limits.",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
limits,
|
||||
usage: Usage::default(),
|
||||
numeric: WorkMeter::new(
|
||||
WorkLimits::new(0, limits.numeric_units)
|
||||
.map_err(|e| error(Span::default(), e.to_string()))?,
|
||||
),
|
||||
failed: None,
|
||||
})
|
||||
}
|
||||
pub fn active(&self) -> Result<(), Error> {
|
||||
self.failed.clone().map_or(Ok(()), Err)
|
||||
}
|
||||
fn charge(
|
||||
&mut self,
|
||||
used: usize,
|
||||
amount: usize,
|
||||
maximum: usize,
|
||||
span: Span,
|
||||
name: &str,
|
||||
) -> Result<(), Error> {
|
||||
self.active()?;
|
||||
if amount > maximum.saturating_sub(used) {
|
||||
let failure = error(span, format!("Source8 {name} limit exceeded."));
|
||||
self.failed = Some(failure.clone());
|
||||
return Err(failure);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn bytes(&mut self, amount: usize, span: Span) -> Result<(), Error> {
|
||||
self.charge(self.usage.bytes, amount, self.limits.bytes, span, "bytes")?;
|
||||
self.usage.bytes += amount;
|
||||
Ok(())
|
||||
}
|
||||
pub fn token(&mut self, span: Span) -> Result<(), Error> {
|
||||
self.charge(self.usage.tokens, 1, self.limits.tokens, span, "tokens")?;
|
||||
self.usage.tokens += 1;
|
||||
Ok(())
|
||||
}
|
||||
pub fn entry(&mut self, span: Span) -> Result<(), Error> {
|
||||
self.charge(
|
||||
self.usage.entries,
|
||||
1,
|
||||
self.limits.entries,
|
||||
span,
|
||||
"syntax entries",
|
||||
)?;
|
||||
self.usage.entries += 1;
|
||||
Ok(())
|
||||
}
|
||||
pub fn reference(&mut self, span: Span) -> Result<(), Error> {
|
||||
self.charge(
|
||||
self.usage.references,
|
||||
1,
|
||||
self.limits.references,
|
||||
span,
|
||||
"references",
|
||||
)?;
|
||||
self.usage.references += 1;
|
||||
Ok(())
|
||||
}
|
||||
pub fn frame(&mut self, depth: usize, span: Span) -> Result<(), Error> {
|
||||
self.charge(0, depth, self.limits.frames, span, "syntax frames")?;
|
||||
self.usage.deepest_frame = self.usage.deepest_frame.max(depth);
|
||||
Ok(())
|
||||
}
|
||||
pub fn exact(&mut self, spelling: &str, span: Span) -> Result<Exact, Error> {
|
||||
self.active()?;
|
||||
match Exact::parse_literal(spelling, &mut self.numeric) {
|
||||
Ok(value) => Ok(value),
|
||||
Err(cause) => {
|
||||
let failure = error(span, cause.to_string());
|
||||
self.failed = Some(failure.clone());
|
||||
Err(failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn numeric_units(&self) -> u64 {
|
||||
self.numeric.usage().numeric_units
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! Raw numeric tokens: finite coordinate checks remain in the existing reader.
|
||||
use super::{Budget, Error, Span, error};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Class {
|
||||
Word,
|
||||
Number,
|
||||
String,
|
||||
Punctuation,
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Token {
|
||||
pub class: Class,
|
||||
pub span: Span,
|
||||
}
|
||||
pub fn tokens(source: &str, budget: &mut Budget) -> Result<Vec<Token>, Error> {
|
||||
budget.bytes(
|
||||
source.len(),
|
||||
Span {
|
||||
start: 0,
|
||||
end: source.len(),
|
||||
},
|
||||
)?;
|
||||
let bytes = source.as_bytes();
|
||||
let mut cursor = 0;
|
||||
let mut result = Vec::new();
|
||||
while cursor < bytes.len() {
|
||||
let start = cursor;
|
||||
// Reserve before scanning/storing a token, including skipped trivia.
|
||||
budget.token(Span { start, end: start })?;
|
||||
let class = if bytes[cursor].is_ascii_whitespace() {
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() {
|
||||
cursor += 1;
|
||||
}
|
||||
None
|
||||
} else if bytes[cursor..].starts_with(b"//") {
|
||||
while cursor < bytes.len() && bytes[cursor] != b'\n' {
|
||||
cursor += 1;
|
||||
}
|
||||
None
|
||||
} else if bytes[cursor].is_ascii_alphabetic() || bytes[cursor] == b'_' {
|
||||
cursor += 1;
|
||||
while cursor < bytes.len()
|
||||
&& (bytes[cursor].is_ascii_alphanumeric() || bytes[cursor] == b'_')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
Some(Class::Word)
|
||||
} else if bytes[cursor].is_ascii_digit() {
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if bytes.get(cursor) == Some(&b'.') {
|
||||
cursor += 1;
|
||||
let fraction = cursor;
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if cursor == fraction {
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Expected decimal fraction digits.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if matches!(bytes.get(cursor), Some(b'e' | b'E')) {
|
||||
cursor += 1;
|
||||
if matches!(bytes.get(cursor), Some(b'+' | b'-')) {
|
||||
cursor += 1;
|
||||
}
|
||||
let exponent = cursor;
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_digit() {
|
||||
cursor += 1;
|
||||
}
|
||||
if cursor == exponent {
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Expected exponent digits.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if bytes
|
||||
.get(cursor)
|
||||
.is_some_and(|b| b.is_ascii_alphabetic() || matches!(b, b'_' | b'.'))
|
||||
{
|
||||
return Err(error(
|
||||
Span { start, end: cursor },
|
||||
"Malformed numeric atom.",
|
||||
));
|
||||
}
|
||||
Some(Class::Number)
|
||||
} else if bytes[cursor] == b'"' {
|
||||
cursor += 1;
|
||||
let mut closed = false;
|
||||
while cursor < bytes.len() {
|
||||
match bytes[cursor] {
|
||||
b'\\' => {
|
||||
cursor += 1;
|
||||
if cursor < bytes.len() {
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
b'"' => {
|
||||
cursor += 1;
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
_ => cursor += 1,
|
||||
}
|
||||
}
|
||||
if !closed {
|
||||
return Err(error(Span { start, end: cursor }, "Unclosed string."));
|
||||
}
|
||||
// Quoted field decoding remains the typed declaration consumer's job.
|
||||
Some(Class::String)
|
||||
} else if b"{}();,[]:=@$+-*/<>!".contains(&bytes[cursor]) {
|
||||
cursor += 1;
|
||||
if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') && bytes.get(cursor) == Some(&b'=')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
Some(Class::Punctuation)
|
||||
} else {
|
||||
let end = start + source[start..].chars().next().unwrap().len_utf8();
|
||||
return Err(error(
|
||||
Span { start, end },
|
||||
"Unexpected expression character.",
|
||||
));
|
||||
};
|
||||
if let Some(class) = class {
|
||||
result.push(Token {
|
||||
class,
|
||||
span: Span { start, end: cursor },
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,550 @@
|
||||
//! Iterative Pratt machine. Source nesting does not recurse on the native stack.
|
||||
use super::*;
|
||||
use lex::{Class, Token};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
enum SequenceKind {
|
||||
List,
|
||||
If,
|
||||
Sum,
|
||||
Unary(UnaryOp),
|
||||
Call(Reference),
|
||||
}
|
||||
struct Sequence {
|
||||
kind: SequenceKind,
|
||||
start: usize,
|
||||
values: Vec<(Option<Reference>, Expression)>,
|
||||
keys: BTreeSet<String>,
|
||||
}
|
||||
enum Task {
|
||||
Expression(u8),
|
||||
Continue(u8),
|
||||
Binary(Expression, BinaryOp),
|
||||
Unary(usize, UnaryOp),
|
||||
Group(usize),
|
||||
Next(Sequence),
|
||||
Collected(Sequence, Option<Reference>),
|
||||
Power(usize),
|
||||
AsKind(usize, Reference),
|
||||
}
|
||||
struct Parser<'a, 'b> {
|
||||
source: &'a str,
|
||||
tokens: &'a [Token],
|
||||
cursor: usize,
|
||||
frames: usize,
|
||||
budget: &'b mut Budget,
|
||||
}
|
||||
pub(super) fn parse(source: &str, tokens: &[Token], budget: &mut Budget) -> Result<Binding, Error> {
|
||||
let mut p = Parser {
|
||||
source,
|
||||
tokens,
|
||||
cursor: 0,
|
||||
frames: 0,
|
||||
budget,
|
||||
};
|
||||
if p.text() == "unbound" {
|
||||
let span = p.span();
|
||||
p.cursor += 1;
|
||||
p.end()?;
|
||||
return Ok(Binding::Unbound(span));
|
||||
}
|
||||
let mut tasks = vec![Task::Expression(0)];
|
||||
let mut values = Vec::new();
|
||||
while let Some(task) = tasks.pop() {
|
||||
p.budget.active()?;
|
||||
match task {
|
||||
Task::Expression(minimum) => {
|
||||
let start = p.span().start;
|
||||
if p.text() != "(" {
|
||||
p.budget.entry(p.span())?;
|
||||
}
|
||||
tasks.push(Task::Continue(minimum));
|
||||
match p.text() {
|
||||
"true" | "false" => {
|
||||
let value = p.text() == "true";
|
||||
let span = p.span();
|
||||
p.cursor += 1;
|
||||
values.push(p.node(Kind::Boolean(value), span)?);
|
||||
}
|
||||
"-" => {
|
||||
p.open()?;
|
||||
p.cursor += 1;
|
||||
tasks.push(Task::Unary(start, UnaryOp::Negate));
|
||||
tasks.push(Task::Expression(7));
|
||||
}
|
||||
"(" => {
|
||||
p.open()?;
|
||||
p.cursor += 1;
|
||||
tasks.push(Task::Group(start));
|
||||
tasks.push(Task::Expression(0));
|
||||
}
|
||||
"[" => {
|
||||
p.open()?;
|
||||
p.cursor += 1;
|
||||
tasks.push(Task::Next(Sequence::new(SequenceKind::List, start)));
|
||||
}
|
||||
"@" => {
|
||||
p.cursor += 1;
|
||||
let reference = p.reference()?;
|
||||
if p.text() == "(" {
|
||||
p.open()?;
|
||||
p.cursor += 1;
|
||||
tasks.push(Task::Next(Sequence::new(
|
||||
SequenceKind::Call(reference),
|
||||
start,
|
||||
)));
|
||||
} else {
|
||||
let end = reference.span.end;
|
||||
values.push(p.node(Kind::Property(reference), Span { start, end })?);
|
||||
}
|
||||
}
|
||||
"$" => {
|
||||
p.cursor += 1;
|
||||
let reference = p.reference()?;
|
||||
let end = reference.span.end;
|
||||
values.push(p.node(Kind::Parameter(reference), Span { start, end })?);
|
||||
}
|
||||
"ratio" => {
|
||||
let (value, literal) = p.ratio()?;
|
||||
values.push(p.number(value, literal)?);
|
||||
}
|
||||
"abs" | "not" | "if" | "sum" => {
|
||||
let kind = match p.text() {
|
||||
"abs" => SequenceKind::Unary(UnaryOp::Absolute),
|
||||
"not" => SequenceKind::Unary(UnaryOp::Not),
|
||||
"if" => SequenceKind::If,
|
||||
_ => SequenceKind::Sum,
|
||||
};
|
||||
p.cursor += 1;
|
||||
p.open()?;
|
||||
p.take("(")?;
|
||||
tasks.push(Task::Next(Sequence::new(kind, start)));
|
||||
}
|
||||
"power" => {
|
||||
p.cursor += 1;
|
||||
p.open()?;
|
||||
p.take("(")?;
|
||||
tasks.push(Task::Power(start));
|
||||
tasks.push(Task::Expression(0));
|
||||
}
|
||||
"as_kind" => {
|
||||
p.cursor += 1;
|
||||
p.open()?;
|
||||
p.take("(")?;
|
||||
p.take("@")?;
|
||||
let kind = p.reference()?;
|
||||
p.take(",")?;
|
||||
tasks.push(Task::AsKind(start, kind));
|
||||
tasks.push(Task::Expression(0));
|
||||
}
|
||||
_ if p
|
||||
.tokens
|
||||
.get(p.cursor)
|
||||
.is_some_and(|token| token.class == Class::Number) =>
|
||||
{
|
||||
let span = p.span();
|
||||
let value = p.budget.exact(&p.source[span.start..span.end], span)?;
|
||||
p.cursor += 1;
|
||||
values.push(p.number(
|
||||
value,
|
||||
LiteralSyntax {
|
||||
span,
|
||||
tokens: vec![span],
|
||||
},
|
||||
)?);
|
||||
}
|
||||
_ => return Err(error(p.span(), "Expected an analysis expression.")),
|
||||
}
|
||||
}
|
||||
Task::Continue(minimum) => {
|
||||
if let Some((op, precedence)) =
|
||||
operator(p.text()).filter(|(_, prec)| *prec >= minimum)
|
||||
{
|
||||
let left = values.pop().expect("completed Pratt operand");
|
||||
if comparison(op) && ungrouped_comparison(&left) {
|
||||
return Err(error(p.span(), "Comparison children require parentheses."));
|
||||
}
|
||||
p.budget.entry(p.span())?;
|
||||
p.cursor += 1;
|
||||
tasks.push(Task::Continue(minimum));
|
||||
tasks.push(Task::Binary(left, op));
|
||||
tasks.push(Task::Expression(precedence + 1));
|
||||
}
|
||||
}
|
||||
Task::Binary(left, op) => {
|
||||
let right = values.pop().expect("completed right operand");
|
||||
if comparison(op) && ungrouped_comparison(&right) {
|
||||
return Err(error(
|
||||
right.span,
|
||||
"Comparison children require parentheses.",
|
||||
));
|
||||
}
|
||||
let span = Span {
|
||||
start: left.span.start,
|
||||
end: right.span.end,
|
||||
};
|
||||
values.push(p.node(
|
||||
Kind::Binary {
|
||||
op,
|
||||
left: Box::new(left),
|
||||
right: Box::new(right),
|
||||
},
|
||||
span,
|
||||
)?);
|
||||
}
|
||||
Task::Unary(start, op) => {
|
||||
let operand = values.pop().expect("completed unary operand");
|
||||
let end = operand.span.end;
|
||||
p.close();
|
||||
values.push(p.node(
|
||||
Kind::Unary {
|
||||
op,
|
||||
operand: Box::new(operand),
|
||||
},
|
||||
Span { start, end },
|
||||
)?);
|
||||
}
|
||||
Task::Group(start) => {
|
||||
let end = p.take(")")?.end;
|
||||
p.close();
|
||||
let value = values.last_mut().expect("completed grouped expression");
|
||||
value.grouped = true;
|
||||
value.span = Span { start, end };
|
||||
}
|
||||
Task::Next(mut sequence) => {
|
||||
let closing = if matches!(sequence.kind, SequenceKind::List) {
|
||||
"]"
|
||||
} else {
|
||||
")"
|
||||
};
|
||||
if p.text() == closing {
|
||||
let end = p.take(closing)?.end;
|
||||
p.close();
|
||||
values.push(p.sequence(sequence, end)?);
|
||||
} else {
|
||||
let count = sequence.values.len();
|
||||
let maximum = match sequence.kind {
|
||||
SequenceKind::List => crate::analysis::MAX_LIST_LENGTH,
|
||||
SequenceKind::Unary(_) => 1,
|
||||
SequenceKind::If => 3,
|
||||
SequenceKind::Sum => 2,
|
||||
SequenceKind::Call(_) => usize::MAX,
|
||||
};
|
||||
if count >= maximum {
|
||||
return Err(error(
|
||||
p.span(),
|
||||
"Too many list members or constructor arguments.",
|
||||
));
|
||||
}
|
||||
let key = if matches!(sequence.kind, SequenceKind::Call(_)) {
|
||||
p.budget.entry(p.span())?;
|
||||
let key = p.reference()?;
|
||||
if !sequence.keys.insert(key.name.clone()) {
|
||||
return Err(error(key.span, "Duplicate call argument name."));
|
||||
}
|
||||
p.take(":")?;
|
||||
Some(key)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
tasks.push(Task::Collected(sequence, key));
|
||||
tasks.push(Task::Expression(0));
|
||||
}
|
||||
}
|
||||
Task::Collected(mut sequence, key) => {
|
||||
sequence
|
||||
.values
|
||||
.push((key, values.pop().expect("completed sequence value")));
|
||||
let closing = if matches!(sequence.kind, SequenceKind::List) {
|
||||
"]"
|
||||
} else {
|
||||
")"
|
||||
};
|
||||
if p.text() != closing {
|
||||
p.take(",")?;
|
||||
if p.text() == closing {
|
||||
return Err(error(p.span(), "Trailing comma is not an expression."));
|
||||
}
|
||||
}
|
||||
tasks.push(Task::Next(sequence));
|
||||
}
|
||||
Task::Power(start) => {
|
||||
let base = values.pop().expect("completed power base");
|
||||
p.take(",")?;
|
||||
let negative = if p.text() == "-" {
|
||||
p.cursor += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let span = p.integer()?;
|
||||
let magnitude = p.source[span.start..span.end]
|
||||
.parse::<u8>()
|
||||
.map_err(|_| error(span, "Power exponent is outside -16..16."))?;
|
||||
if magnitude > 16 {
|
||||
return Err(error(span, "Power exponent is outside -16..16."));
|
||||
}
|
||||
let exponent = if negative {
|
||||
-(magnitude as i8)
|
||||
} else {
|
||||
magnitude as i8
|
||||
};
|
||||
let end = p.take(")")?.end;
|
||||
p.close();
|
||||
values.push(p.node(
|
||||
Kind::Power {
|
||||
base: Box::new(base),
|
||||
exponent,
|
||||
},
|
||||
Span { start, end },
|
||||
)?);
|
||||
}
|
||||
Task::AsKind(start, kind) => {
|
||||
let value = values.pop().expect("completed kind assertion value");
|
||||
let end = p.take(")")?.end;
|
||||
p.close();
|
||||
values.push(p.node(
|
||||
Kind::AsKind {
|
||||
kind,
|
||||
value: Box::new(value),
|
||||
},
|
||||
Span { start, end },
|
||||
)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
p.end()?;
|
||||
Ok(Binding::Expression(
|
||||
values.pop().expect("one complete expression"),
|
||||
))
|
||||
}
|
||||
impl Sequence {
|
||||
fn new(kind: SequenceKind, start: usize) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
start,
|
||||
values: Vec::new(),
|
||||
keys: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Parser<'_, '_> {
|
||||
fn span(&self) -> Span {
|
||||
self.tokens.get(self.cursor).map_or(
|
||||
Span {
|
||||
start: self.source.len(),
|
||||
end: self.source.len(),
|
||||
},
|
||||
|token| token.span,
|
||||
)
|
||||
}
|
||||
fn text(&self) -> &str {
|
||||
let span = self.span();
|
||||
&self.source[span.start..span.end]
|
||||
}
|
||||
fn take(&mut self, expected: &str) -> Result<Span, Error> {
|
||||
if self.text() != expected {
|
||||
return Err(error(self.span(), format!("Expected '{expected}'.")));
|
||||
}
|
||||
let span = self.span();
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
fn end(&self) -> Result<(), Error> {
|
||||
if self.cursor != self.tokens.len() {
|
||||
Err(error(
|
||||
self.span(),
|
||||
"Unexpected content after analysis expression.",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn open(&mut self) -> Result<(), Error> {
|
||||
self.budget.frame(self.frames + 1, self.span())?;
|
||||
self.frames += 1;
|
||||
Ok(())
|
||||
}
|
||||
fn close(&mut self) {
|
||||
self.frames -= 1;
|
||||
}
|
||||
fn reference(&mut self) -> Result<Reference, Error> {
|
||||
let span = self.span();
|
||||
if !self
|
||||
.tokens
|
||||
.get(self.cursor)
|
||||
.is_some_and(|token| token.class == Class::Word)
|
||||
|| !crate::valid_alias(self.text())
|
||||
{
|
||||
return Err(error(span, "Expected a valid reference or parameter name."));
|
||||
}
|
||||
self.budget.reference(span)?;
|
||||
let name = self.text().to_owned();
|
||||
self.cursor += 1;
|
||||
Ok(Reference { name, span })
|
||||
}
|
||||
fn integer(&mut self) -> Result<Span, Error> {
|
||||
let span = self.span();
|
||||
let text = self.text();
|
||||
if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(error(span, "Expected unsigned integer digits."));
|
||||
}
|
||||
if text.len() > crate::analysis_numeric::MAX_COMPONENT_DIGITS {
|
||||
return Err(error(span, "Integer component exceeds digit limit."));
|
||||
}
|
||||
self.cursor += 1;
|
||||
Ok(span)
|
||||
}
|
||||
fn ratio(&mut self) -> Result<(Exact, LiteralSyntax), Error> {
|
||||
let first_token = self.cursor;
|
||||
let start = self.take("ratio")?.start;
|
||||
self.open()?;
|
||||
self.take("(")?;
|
||||
let negative = if self.text() == "-" {
|
||||
self.cursor += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let numerator = self.integer()?;
|
||||
self.take(",")?;
|
||||
let denominator = self.integer()?;
|
||||
let end = self.take(")")?.end;
|
||||
self.close();
|
||||
let n = &self.source[numerator.start..numerator.end];
|
||||
let d = &self.source[denominator.start..denominator.end];
|
||||
let length = 9 + usize::from(negative) + n.len() + d.len();
|
||||
if length > crate::analysis_numeric::MAX_LITERAL_BYTES {
|
||||
return Err(error(
|
||||
Span { start, end },
|
||||
"Numeric reconstruction exceeds literal byte limit.",
|
||||
));
|
||||
}
|
||||
let normalized = format!("ratio({}{n}, {d})", if negative { "-" } else { "" });
|
||||
Ok((
|
||||
self.budget.exact(&normalized, Span { start, end })?,
|
||||
LiteralSyntax {
|
||||
span: Span { start, end },
|
||||
tokens: self.tokens[first_token..self.cursor]
|
||||
.iter()
|
||||
.map(|token| token.span)
|
||||
.collect(),
|
||||
},
|
||||
))
|
||||
}
|
||||
fn number(&mut self, magnitude: Exact, literal: LiteralSyntax) -> Result<Expression, Error> {
|
||||
let Span { start, end } = literal.span;
|
||||
let mut expression = if self.text() == "[" {
|
||||
self.open()?;
|
||||
self.cursor += 1;
|
||||
self.take("@")?;
|
||||
let unit = self.reference()?;
|
||||
let end = self.take("]")?.end;
|
||||
self.close();
|
||||
self.node(Kind::Quantity { magnitude, unit }, Span { start, end })
|
||||
} else {
|
||||
self.node(Kind::Number(magnitude), Span { start, end })
|
||||
}?;
|
||||
expression.literal = Some(literal);
|
||||
Ok(expression)
|
||||
}
|
||||
fn node(&mut self, kind: Kind, span: Span) -> Result<Expression, Error> {
|
||||
let child_depth = match &kind {
|
||||
Kind::Unary { operand, .. } => operand.depth,
|
||||
Kind::Binary { left, right, .. } => left.depth.max(right.depth),
|
||||
Kind::If {
|
||||
condition,
|
||||
when_true,
|
||||
when_false,
|
||||
} => condition.depth.max(when_true.depth).max(when_false.depth),
|
||||
Kind::List(values) => values.iter().map(|v| v.depth).max().unwrap_or(0),
|
||||
Kind::Call { arguments, .. } => {
|
||||
arguments.iter().map(|a| a.value.depth).max().unwrap_or(0)
|
||||
}
|
||||
Kind::Sum { values, initial } => values.depth.max(initial.depth),
|
||||
Kind::Power { base, .. } => base.depth,
|
||||
Kind::AsKind { value, .. } => value.depth,
|
||||
_ => 0,
|
||||
};
|
||||
if child_depth >= crate::analysis::MAX_EXPRESSION_DEPTH {
|
||||
return Err(error(span, "Expression logical depth exceeds 128."));
|
||||
}
|
||||
Ok(Expression {
|
||||
kind,
|
||||
span,
|
||||
depth: child_depth + 1,
|
||||
grouped: false,
|
||||
literal: None,
|
||||
})
|
||||
}
|
||||
fn sequence(&mut self, sequence: Sequence, end: usize) -> Result<Expression, Error> {
|
||||
let span = Span {
|
||||
start: sequence.start,
|
||||
end,
|
||||
};
|
||||
let mut values = sequence.values.into_iter();
|
||||
let mut next = || {
|
||||
values
|
||||
.next()
|
||||
.map(|(_, value)| Box::new(value))
|
||||
.ok_or_else(|| error(span, "Missing required constructor argument."))
|
||||
};
|
||||
let kind = match sequence.kind {
|
||||
SequenceKind::If => Kind::If {
|
||||
condition: next()?,
|
||||
when_true: next()?,
|
||||
when_false: next()?,
|
||||
},
|
||||
SequenceKind::Sum => Kind::Sum {
|
||||
values: next()?,
|
||||
initial: next()?,
|
||||
},
|
||||
SequenceKind::Unary(op) => Kind::Unary {
|
||||
op,
|
||||
operand: next()?,
|
||||
},
|
||||
SequenceKind::List => Kind::List(values.map(|(_, value)| value).collect()),
|
||||
SequenceKind::Call(definition) => Kind::Call {
|
||||
definition,
|
||||
arguments: values
|
||||
.map(|(parameter, value)| Argument {
|
||||
parameter: parameter.expect("parsed named argument"),
|
||||
value,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
};
|
||||
self.node(kind, span)
|
||||
}
|
||||
}
|
||||
fn comparison(op: BinaryOp) -> bool {
|
||||
matches!(
|
||||
op,
|
||||
BinaryOp::Equal
|
||||
| BinaryOp::NotEqual
|
||||
| BinaryOp::Less
|
||||
| BinaryOp::LessOrEqual
|
||||
| BinaryOp::Greater
|
||||
| BinaryOp::GreaterOrEqual
|
||||
)
|
||||
}
|
||||
fn ungrouped_comparison(value: &Expression) -> bool {
|
||||
!value.grouped && matches!(&value.kind, Kind::Binary { op, .. } if comparison(*op))
|
||||
}
|
||||
fn operator(text: &str) -> Option<(BinaryOp, u8)> {
|
||||
Some(match text {
|
||||
"or" => (BinaryOp::Or, 1),
|
||||
"and" => (BinaryOp::And, 2),
|
||||
"==" => (BinaryOp::Equal, 3),
|
||||
"!=" => (BinaryOp::NotEqual, 3),
|
||||
"<" => (BinaryOp::Less, 4),
|
||||
"<=" => (BinaryOp::LessOrEqual, 4),
|
||||
">" => (BinaryOp::Greater, 4),
|
||||
">=" => (BinaryOp::GreaterOrEqual, 4),
|
||||
"+" => (BinaryOp::Add, 5),
|
||||
"-" => (BinaryOp::Subtract, 5),
|
||||
"*" => (BinaryOp::Multiply, 6),
|
||||
"/" => (BinaryOp::Divide, 6),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
use super::*;
|
||||
use crate::language::ParseLimits;
|
||||
use budget::{Budget, Limits};
|
||||
|
||||
fn limits() -> Limits {
|
||||
Limits::source8(ParseLimits::UNLIMITED)
|
||||
}
|
||||
fn budget() -> Budget {
|
||||
Budget::new(limits()).unwrap()
|
||||
}
|
||||
fn expression(source: &str) -> Expression {
|
||||
match parse(source, &mut budget()).unwrap() {
|
||||
Binding::Expression(value) => value,
|
||||
Binding::Unbound(_) => panic!("expected expression"),
|
||||
}
|
||||
}
|
||||
fn number(value: &Expression) -> String {
|
||||
let Kind::Number(exact) = &value.kind else {
|
||||
panic!("expected Number");
|
||||
};
|
||||
exact.canonical_literal()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn significant_ratio_and_quantity_trivia_preserve_original_spans() {
|
||||
let source = "ratio // explanation café\n ( - // sign\n 3, // denominator\n 4 ) // measurement\n [@millisecond]";
|
||||
let parsed = expression(source);
|
||||
assert_eq!(
|
||||
parsed.span,
|
||||
Span {
|
||||
start: 0,
|
||||
end: source.len()
|
||||
}
|
||||
);
|
||||
let literal = parsed.literal.as_ref().unwrap();
|
||||
assert!(source[literal.span.start..literal.span.end].contains("// denominator"));
|
||||
assert_eq!(
|
||||
literal
|
||||
.tokens
|
||||
.iter()
|
||||
.map(|span| &source[span.start..span.end])
|
||||
.collect::<Vec<_>>(),
|
||||
["ratio", "(", "-", "3", ",", "4", ")"]
|
||||
);
|
||||
let Kind::Quantity { magnitude, unit } = parsed.kind else {
|
||||
panic!("expected Quantity");
|
||||
};
|
||||
assert_eq!(magnitude.canonical_literal(), "ratio(-3, 4)");
|
||||
assert_eq!(unit.name, "millisecond");
|
||||
assert_eq!(&source[unit.span.start..unit.span.end], "millisecond");
|
||||
let mut renamed = source.to_owned();
|
||||
renamed.replace_range(unit.span.start..unit.span.end, "ms");
|
||||
assert_eq!(renamed, source.replacen("millisecond", "ms", 1));
|
||||
assert_eq!(
|
||||
number(&expression("ratio (1, // rationale\n 2)")),
|
||||
"ratio(1, 2)"
|
||||
);
|
||||
for invalid in [
|
||||
"1 e - 2",
|
||||
"1e // x\n -2",
|
||||
"1 . 2",
|
||||
"ratio(1 + 2, 3)",
|
||||
"ratio(1, -2)",
|
||||
"ratio(+1, 2)",
|
||||
"ratio(1.0, 2)",
|
||||
"ratio(1, 0)",
|
||||
] {
|
||||
assert!(parse(invalid, &mut budget()).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_numeric_lexing_preserves_exact_atoms_and_operator_boundaries() {
|
||||
let source = "1-2 1e-2 ratio(-1,2) -(1) <= >= == !=";
|
||||
let tokens = lex::tokens(source, &mut budget()).unwrap();
|
||||
let text: Vec<_> = tokens
|
||||
.iter()
|
||||
.map(|t| &source[t.span.start..t.span.end])
|
||||
.collect();
|
||||
assert_eq!(
|
||||
text,
|
||||
[
|
||||
"1", "-", "2", "1e-2", "ratio", "(", "-", "1", ",", "2", ")", "-", "(", "1", ")", "<=",
|
||||
">=", "==", "!="
|
||||
]
|
||||
);
|
||||
let huge = "9".repeat(400);
|
||||
assert_eq!(
|
||||
lex::tokens(&huge, &mut budget()).unwrap()[0].class,
|
||||
lex::Class::Number
|
||||
);
|
||||
assert_eq!(number(&expression(&huge)), huge);
|
||||
for invalid in ["1e", "1e+", "1_0", "0x1", "1.2.3"] {
|
||||
assert!(lex::tokens(invalid, &mut budget()).is_err(), "{invalid}");
|
||||
}
|
||||
let quoted = "\"quoted\\\" café\"";
|
||||
let tokens = lex::tokens(quoted, &mut budget()).unwrap();
|
||||
assert_eq!(tokens[0].class, lex::Class::String);
|
||||
assert_eq!("ed[tokens[0].span.start..tokens[0].span.end], quoted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unary_literals_precedence_and_comparison_children_keep_distinct_trees() {
|
||||
assert_eq!(number(&expression("ratio(-1, 1)")), "ratio(-1, 1)");
|
||||
let negative = expression("-(1)");
|
||||
assert!(matches!(
|
||||
negative.kind,
|
||||
Kind::Unary {
|
||||
op: UnaryOp::Negate,
|
||||
..
|
||||
}
|
||||
));
|
||||
let left = expression("(1 - 2) - 3");
|
||||
let right = expression("1 - (2 - 3)");
|
||||
let Kind::Binary {
|
||||
left, right: last, ..
|
||||
} = left.kind
|
||||
else {
|
||||
panic!();
|
||||
};
|
||||
assert!(matches!(
|
||||
left.kind,
|
||||
Kind::Binary {
|
||||
op: BinaryOp::Subtract,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(number(&last), "3");
|
||||
let Kind::Binary {
|
||||
left: first, right, ..
|
||||
} = right.kind
|
||||
else {
|
||||
panic!();
|
||||
};
|
||||
assert_eq!(number(&first), "1");
|
||||
assert!(matches!(
|
||||
right.kind,
|
||||
Kind::Binary {
|
||||
op: BinaryOp::Subtract,
|
||||
..
|
||||
}
|
||||
));
|
||||
for valid in [
|
||||
"(1 < 2) == true",
|
||||
"(true == false) == true",
|
||||
"true == (1 < 2)",
|
||||
"1 + 2 * 3",
|
||||
"not(false) and true or false",
|
||||
] {
|
||||
parse(valid, &mut budget()).unwrap();
|
||||
}
|
||||
for invalid in [
|
||||
"1 < 2 == true",
|
||||
"true == false == true",
|
||||
"1 < 2 < 3",
|
||||
"true == 1 < 2",
|
||||
] {
|
||||
assert!(parse(invalid, &mut budget()).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_expression_shape_and_exact_constructor_arity_is_represented() {
|
||||
let tree = expression("if(true, sum([@p, $x], 0), as_kind(@duration, power(abs(-(2)), -2)))");
|
||||
let Kind::If {
|
||||
condition,
|
||||
when_true,
|
||||
when_false,
|
||||
} = tree.kind
|
||||
else {
|
||||
panic!();
|
||||
};
|
||||
assert_eq!(condition.kind, Kind::Boolean(true));
|
||||
let Kind::Sum { values, initial } = when_true.kind else {
|
||||
panic!();
|
||||
};
|
||||
assert_eq!(number(&initial), "0");
|
||||
let Kind::List(values) = values.kind else {
|
||||
panic!();
|
||||
};
|
||||
assert!(matches!(&values[0].kind, Kind::Property(r) if r.name == "p"));
|
||||
assert!(matches!(&values[1].kind, Kind::Parameter(r) if r.name == "x"));
|
||||
let Kind::AsKind { kind, value } = when_false.kind else {
|
||||
panic!();
|
||||
};
|
||||
assert_eq!(kind.name, "duration");
|
||||
assert!(matches!(value.kind, Kind::Power { exponent: -2, .. }));
|
||||
assert!(matches!(expression("[]").kind, Kind::List(v) if v.is_empty()));
|
||||
assert!(
|
||||
matches!(expression("@zero()").kind, Kind::Call { arguments, .. } if arguments.is_empty())
|
||||
);
|
||||
assert!(matches!(
|
||||
parse(" unbound // pending", &mut budget()).unwrap(),
|
||||
Binding::Unbound(_)
|
||||
));
|
||||
for invalid in [
|
||||
"sum([1])",
|
||||
"sum([1], 0, 2)",
|
||||
"if(true, 1)",
|
||||
"abs()",
|
||||
"not(true, false)",
|
||||
"power(2, 17)",
|
||||
"power(2, 1.0)",
|
||||
"as_kind(duration, 1)",
|
||||
"[1,]",
|
||||
"[1 2]",
|
||||
"(1+2)[@unit]",
|
||||
] {
|
||||
assert!(parse(invalid, &mut budget()).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_call_argument_occurrences_have_distinct_original_spans() {
|
||||
let source = "@combine(a: @p, b: 2) + @combine(a: @q, b: 4)";
|
||||
let tree = expression(source);
|
||||
let Kind::Binary { left, right, .. } = tree.kind else {
|
||||
panic!();
|
||||
};
|
||||
let Kind::Call {
|
||||
definition: first,
|
||||
arguments: a,
|
||||
} = left.kind
|
||||
else {
|
||||
panic!();
|
||||
};
|
||||
let Kind::Call {
|
||||
definition: second,
|
||||
arguments: b,
|
||||
} = right.kind
|
||||
else {
|
||||
panic!();
|
||||
};
|
||||
assert_eq!(first.name, second.name);
|
||||
assert_ne!(first.span, second.span);
|
||||
assert_eq!(a[0].parameter.name, b[0].parameter.name);
|
||||
assert_ne!(a[0].parameter.span, b[0].parameter.span);
|
||||
for argument in a.iter().chain(&b) {
|
||||
assert_eq!(
|
||||
&source[argument.parameter.span.start..argument.parameter.span.end],
|
||||
argument.parameter.name
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
parse("@combine(a: 1, a: 2)", &mut budget())
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("Duplicate")
|
||||
);
|
||||
assert!(parse("@combine(a: 1, b:)", &mut budget()).is_err());
|
||||
// Resolution to owner-qualified ParameterId and actual source rewrites are
|
||||
// deliberately a later stage; this oracle establishes unique syntax spans.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budgets_aggregate_across_fragments_and_latch_numeric_exhaustion() {
|
||||
let mut first = budget();
|
||||
parse("ratio(1, 3)", &mut first).unwrap();
|
||||
let cost = first.numeric_units();
|
||||
assert!(cost > 0);
|
||||
let mut settings = limits();
|
||||
settings.numeric_units = cost;
|
||||
let mut shared = Budget::new(settings).unwrap();
|
||||
parse("ratio (1, // same numeric cost\n 3)", &mut shared).unwrap();
|
||||
assert_eq!(shared.numeric_units(), cost);
|
||||
let failure = parse("2", &mut shared).unwrap_err();
|
||||
let usage = shared.usage;
|
||||
assert_eq!(parse("unbound", &mut shared).unwrap_err(), failure);
|
||||
assert_eq!(shared.usage, usage);
|
||||
parse("2", &mut budget()).unwrap();
|
||||
|
||||
let mut settings = limits();
|
||||
settings.entries = 2;
|
||||
let mut shared = Budget::new(settings).unwrap();
|
||||
parse("true", &mut shared).unwrap();
|
||||
parse("false", &mut shared).unwrap();
|
||||
assert!(
|
||||
parse("true", &mut shared)
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("entries")
|
||||
);
|
||||
assert_eq!(shared.usage.entries, 2);
|
||||
let mut settings = limits();
|
||||
settings.references = 1;
|
||||
let mut shared = Budget::new(settings).unwrap();
|
||||
parse("@one", &mut shared).unwrap();
|
||||
assert!(
|
||||
parse("$two", &mut shared)
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("references")
|
||||
);
|
||||
assert_eq!(shared.usage.references, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_token_frame_logical_and_list_boundaries_have_nearby_successes() {
|
||||
let mut settings = limits();
|
||||
settings.bytes = 4;
|
||||
let mut shared = Budget::new(settings).unwrap();
|
||||
parse("true", &mut shared).unwrap();
|
||||
assert!(
|
||||
parse(" ", &mut shared)
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("bytes")
|
||||
);
|
||||
assert_eq!(shared.usage.bytes, 4);
|
||||
let mut settings = limits();
|
||||
settings.tokens = 2;
|
||||
let mut shared = Budget::new(settings).unwrap();
|
||||
parse("true ", &mut shared).unwrap();
|
||||
assert!(
|
||||
parse("// note", &mut shared)
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("tokens")
|
||||
);
|
||||
assert_eq!(shared.usage.tokens, 2);
|
||||
let nested = |depth: usize| format!("{}true{}", "(".repeat(depth), ")".repeat(depth));
|
||||
let mut shared = budget();
|
||||
assert_eq!(
|
||||
match parse(&nested(256), &mut shared).unwrap() {
|
||||
Binding::Expression(e) => e.depth,
|
||||
_ => 0,
|
||||
},
|
||||
1
|
||||
);
|
||||
assert_eq!(shared.usage.deepest_frame, 256);
|
||||
assert!(
|
||||
parse(&nested(257), &mut budget())
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("frames")
|
||||
);
|
||||
let unary = |depth: usize| format!("{}true{}", "not(".repeat(depth - 1), ")".repeat(depth - 1));
|
||||
assert_eq!(expression(&unary(128)).depth, 128);
|
||||
assert!(
|
||||
parse(&unary(129), &mut budget())
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("logical depth")
|
||||
);
|
||||
let list = |count: usize| format!("[{}]", vec!["true"; count].join(","));
|
||||
parse(&list(1024), &mut budget()).unwrap();
|
||||
assert!(parse(&list(1025), &mut budget()).is_err());
|
||||
let lowered = Limits::source8(ParseLimits {
|
||||
max_source_bytes: 8,
|
||||
max_graph_objects: 3,
|
||||
max_records: 5,
|
||||
max_step_references: 2,
|
||||
});
|
||||
assert_eq!(
|
||||
(
|
||||
lowered.bytes,
|
||||
lowered.tokens,
|
||||
lowered.entries,
|
||||
lowered.references
|
||||
),
|
||||
(8, 8, 5, 2)
|
||||
);
|
||||
let mut above = limits();
|
||||
above.frames += 1;
|
||||
assert!(Budget::new(above).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_wide_and_deep_input_cleanup_does_not_poison_a_fresh_parse() {
|
||||
let source = "@f(a: if(true, [ratio (1, //why\n 2), @p], sum([1, 2], 0)), b: power(2, -3))";
|
||||
let tokens = lex::tokens(source, &mut budget()).unwrap();
|
||||
for token in tokens.iter().filter(|token| {
|
||||
matches!(
|
||||
&source[token.span.start..token.span.end],
|
||||
"(" | "[" | "," | ":" | "@"
|
||||
)
|
||||
}) {
|
||||
assert!(parse(&source[..token.span.end], &mut budget()).is_err());
|
||||
assert_eq!(number(&expression("ratio(2, 3)")), "ratio(2, 3)");
|
||||
}
|
||||
for malformed in [
|
||||
format!("{}true", "(".repeat(256)),
|
||||
format!("[{}", vec!["true"; 1024].join(",")),
|
||||
"-".repeat(10_000),
|
||||
] {
|
||||
assert!(parse(&malformed, &mut budget()).is_err());
|
||||
parse("true", &mut budget()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_source_headers_and_existing_numeric_token_controls_are_unchanged() {
|
||||
assert!(crate::language::parse("arcadia 8; project p \"P\" {}").is_err());
|
||||
for accepted in ["-1.5", "1e-2", "0", "-0"] {
|
||||
let tokens = crate::language::tokenize(accepted);
|
||||
assert_eq!(tokens.len(), 1);
|
||||
assert_eq!(tokens[0].class, crate::language::TokenClass::Number);
|
||||
assert_eq!(
|
||||
tokens[0].span,
|
||||
Span {
|
||||
start: 0,
|
||||
end: accepted.len()
|
||||
}
|
||||
);
|
||||
}
|
||||
for rejected in ["+1", "1_0", "0x1", "1e"] {
|
||||
assert!(
|
||||
crate::language::tokenize(rejected)
|
||||
.iter()
|
||||
.any(|t| t.class == crate::language::TokenClass::Error)
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user