sorokin.oleg.edu/ARCADIA-Studio
1
0
Fork 0
Code Issues 37 Pull Requests 2 Packages Projects Releases 9 Wiki Activity

Add private mixed bundle syntax witness over shared parser and traversal

Browse Source
This commit is contained in:
sorokin.oleg.edu
2026-09-23 01:25:00 -07:00
parent 10d7f3340d
commit 0fc0826435
6 changed files with 1759 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/oa-core/src/language.rs
+178 -28
View File
@@ -1,8 +1,8 @@
//! Typed OA/SA language, tolerant source analysis, and explicit V1/V2/V3/V4 migration.
mod raw_tokens;
#[path = "language/semantic_source.rs"]
mod semantic_source;
mod source3_bridge;
mod raw_tokens;
// Staged syntax foundation only. Public source8 admission waits for complete
// declaration lowering, source updates and the root-owned resource gate.
#[cfg(test)]
@@ -480,6 +480,8 @@ fn parse_document(source: &str, limits: ParseLimits) -> Result<ParsedDocument, V
cursor: 0,
value_depth: 0,
limits,
#[cfg(test)]
mixed: None,
}
.document()
.map_err(|e| vec![diagnostic(source, e)])
@@ -914,6 +916,8 @@ pub(crate) fn parse_unit(
cursor: 0,
value_depth: 0,
limits,
#[cfg(test)]
mixed: None,
}
.unit(entry, true)
.map_err(|e| Box::new(diagnostic(source, e)))
@@ -1005,31 +1009,52 @@ fn edit_positions(
unit: SourceUnit,
) -> Result<Vec<(Id, EditDeclaration)>, Box<Diagnostic>> {
let tokens = strict_tokens(source).map_err(|e| Box::new(diagnostic(source, e)))?;
let mut ends: Vec<_> = tokens.iter().map(|token| token.span.end).collect();
edit_positions_from_tokens(source, &unit, |at| tokens[at], tokens.len(), None)
}
fn edit_positions_from_tokens(
source: &str,
unit: &SourceUnit,
token: impl Fn(usize) -> Lexeme,
count: usize,
value_ends: Option<&BTreeMap<usize, usize>>,
) -> Result<Vec<(Id, EditDeclaration)>, Box<Diagnostic>> {
let mut ends: Vec<_> = (0..count).map(|at| token(at).span.end).collect();
let mut stack = Vec::new();
for (index, token) in tokens.iter().enumerate() {
for index in 0..count {
let token = token(index);
match token.text(source) {
"{" | "[" => stack.push(index),
"}" | "]" => ends[stack.pop().expect("parser checked delimiters")] = token.span.end,
_ => {}
}
}
struct Positions<'a> {
tokens: &'a [Lexeme],
struct Positions<'a, F> {
token_at: F,
ends: Vec<usize>,
value_ends: Option<&'a BTreeMap<usize, usize>>,
}
impl Positions<'_> {
impl<F: Fn(usize) -> Lexeme> Positions<'_, F> {
fn token(&self, start: usize) -> usize {
self.tokens
.partition_point(|token| token.span.start < start)
let mut low = 0;
let mut high = self.ends.len();
while low < high {
let middle = low + (high - low) / 2;
if (self.token_at)(middle).span.start < start {
low = middle + 1;
} else {
high = middle;
}
}
low
}
fn fields(&self, fields: &BTreeMap<String, Ast>) -> BTreeMap<String, EditField> {
fields
.iter()
.map(|(key, ast)| {
let value = self.value(ast);
let start = self.tokens[self.token(value.span.start) - 1].span.start;
let end = self.tokens[self.token(value.span.end)].span.end;
let start = (self.token_at)(self.token(value.span.start) - 1).span.start;
let end = (self.token_at)(self.token(value.span.end)).span.end;
(
key.clone(),
EditField {
@@ -1044,10 +1069,15 @@ fn edit_positions(
let index = self.token(ast.offset());
let span = Span {
start: ast.offset(),
end: match ast {
Ast::Reference(..) | Ast::HistoricalUuid(..) => self.tokens[index + 1].span.end,
_ => self.ends[index],
},
end: self
.value_ends
.and_then(|ends| ends.get(&ast.offset()).copied())
.unwrap_or_else(|| match ast {
Ast::Reference(..) | Ast::HistoricalUuid(..) => {
(self.token_at)(index + 1).span.end
}
_ => self.ends[index],
}),
};
let kind = match ast {
Ast::Object(fields, _) => EditKind::Record(self.fields(fields)),
@@ -1062,11 +1092,12 @@ fn edit_positions(
}
}
let positions = Positions {
tokens: &tokens,
token_at: &token,
ends,
value_ends,
};
unit.declarations
.into_iter()
.iter()
.map(|declaration| {
if declaration.id_insertion.is_some() {
return Err(Box::new(diagnostic(
@@ -1086,8 +1117,8 @@ fn edit_positions(
let start = positions.token(declaration.span.start);
let header = roles
.iter()
.zip(&tokens[start..])
.map(|(role, token)| (*role, token.span))
.zip(start..count)
.map(|(role, at)| (*role, token(at).span))
.collect();
Ok((
declaration.id,
@@ -1114,10 +1145,35 @@ struct Parser<'a> {
cursor: usize,
value_depth: usize,
limits: ParseLimits,
#[cfg(test)]
mixed: Option<analysis_source::mixed::State<'a>>,
}
impl Parser<'_> {
fn token(&self, index: usize) -> Option<Lexeme> {
#[cfg(test)]
if let Some(mixed) = &self.mixed {
return mixed.token(index);
}
self.tokens.get(index).copied()
}
fn current(&self) -> Lexeme {
self.tokens[self.cursor.min(self.tokens.len() - 1)]
self.token(self.cursor).unwrap_or(Lexeme {
class: TokenClass::Punctuation,
span: Span {
start: self.source.len(),
end: self.source.len(),
},
})
}
fn syntax_entry(&mut self) -> Result<(), Error> {
#[cfg(test)]
{
let span = self.current().span;
if let Some(mixed) = &mut self.mixed {
mixed.entry(span)?;
}
}
Ok(())
}
fn text(&self) -> &str {
self.current().text(self.source)
@@ -1154,6 +1210,10 @@ impl Parser<'_> {
}
fn optional_id(&mut self) -> Result<Id, Error> {
if self.text() != "id" {
#[cfg(test)]
if self.mixed.is_some() {
return Err(self.error("Private mixed syntax requires explicit IDs."));
}
return Ok(Id::new_v4());
}
self.cursor += 1;
@@ -1194,6 +1254,7 @@ impl Parser<'_> {
Ok(fields)
}
fn field(&mut self) -> Result<(String, Ast, Span), Error> {
self.syntax_entry()?;
let key = self.text().to_owned();
if !matches!(
self.current().class,
@@ -1204,15 +1265,19 @@ impl Parser<'_> {
self.cursor += 1;
let start = self.current().span.start;
let value = self.value()?;
let end = self.tokens[self.cursor - 1].span.end;
let end = self
.token(self.cursor - 1)
.expect("consumed token")
.span
.end;
self.take(";")?;
Ok((key, value, Span { start, end }))
}
fn nested_header_kind(&self) -> Option<DeclKind> {
let kind = declaration_kind(self.text())?;
let alias = self.tokens.get(self.cursor + 1)?;
let following = self.tokens.get(self.cursor + 2)?;
let alias = self.token(self.cursor + 1)?;
let following = self.token(self.cursor + 2)?;
// A node's ordinary `diagram @view;` field is not a declaration.
if !crate::valid_alias(alias.text(self.source)) {
return None;
@@ -1227,14 +1292,37 @@ impl Parser<'_> {
if self.value_depth >= 64 {
return Err(self.error("Typed values may not nest more than 64 levels."));
}
self.syntax_entry()?;
self.value_depth += 1;
let result = self.value_inner();
self.value_depth -= 1;
#[cfg(test)]
if let Ok(ast) = &result {
let end = self
.token(self.cursor - 1)
.expect("consumed value")
.span
.end;
if let Some(mixed) = &mut self.mixed {
mixed.value_ends.insert(ast.offset(), end);
}
}
result
}
fn value_inner(&mut self) -> Result<Ast, Error> {
let token = self.current();
let offset = token.span.start;
#[cfg(test)]
if self.mixed.is_some()
&& (token.class == TokenClass::Number || matches!(self.text(), "-" | "+"))
{
let value = self
.mixed
.as_ref()
.expect("mixed view")
.ordinary_number(&mut self.cursor)?;
return Ok(Ast::Literal(value, offset));
}
match self.text() {
"uuid" => {
self.cursor += 1;
@@ -1332,16 +1420,31 @@ impl Parser<'_> {
fn unit(&mut self, entry: bool, includes_allowed: bool) -> Result<SourceUnit, Error> {
let mut header = if entry {
self.take("arcadia")?;
if !matches!(self.text(), "3" | "4" | "5" | "6" | "7") {
return Err(
self.error("Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.")
);
#[cfg(test)]
let mixed_mode = self.mixed.is_some();
#[cfg(not(test))]
let mixed_mode = false;
if if mixed_mode {
self.text() != "8"
} else {
!matches!(self.text(), "3" | "4" | "5" | "6" | "7")
} {
return Err(self.error(if mixed_mode {
"Private mixed syntax requires arcadia 8."
} else {
"Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7."
}));
}
let schema_version = self.text().parse().expect("checked source version");
self.cursor += 1;
self.take(";")?;
let start = self.current().span.start;
self.take("project")?;
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.record(false, Span { start, end: start })?;
}
self.syntax_entry()?;
let alias = self.identifier()?;
let name = self.quoted()?;
let id_insertion = (self.text() != "id").then_some(self.current().span.start);
@@ -1382,6 +1485,17 @@ impl Parser<'_> {
let mut open: Vec<usize> = Vec::new();
loop {
let offset = self.current().span.start;
#[cfg(test)]
if let Some(mixed) = &self.mixed {
if mixed.typed_header(self.cursor, open.is_empty()) {
let owner = open.last().map(|&at| declarations[at].id);
self.mixed
.as_mut()
.expect("mixed view")
.parse_typed(&mut self.cursor, owner)?;
continue;
}
}
if let Some(&owner) = open.last() {
if self.text() == "}" {
declarations[owner].body.end = offset;
@@ -1425,6 +1539,14 @@ impl Parser<'_> {
if !includes_allowed {
return Err(self.error("An include requires a source bundle; single-string parsing never reads files."));
}
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.include(Span {
start: offset,
end: offset,
})?;
}
self.syntax_entry()?;
self.cursor += 1;
let path = self.quoted()?;
self.take(";")?;
@@ -1432,7 +1554,11 @@ impl Parser<'_> {
path,
Span {
start: offset,
end: self.tokens[self.cursor - 1].span.end,
end: self
.token(self.cursor - 1)
.expect("consumed token")
.span
.end,
},
));
continue;
@@ -1444,20 +1570,40 @@ impl Parser<'_> {
if entry_header.description.is_some() {
return Err(self.error("Duplicate project description."));
}
self.syntax_entry()?;
self.cursor += 1;
let description = self.quoted()?;
self.take(";")?;
entry_header.project.description = description;
entry_header.description = Some(Span {
start: offset,
end: self.tokens[self.cursor - 1].span.end,
end: self
.token(self.cursor - 1)
.expect("consumed token")
.span
.end,
});
header = Some(entry_header);
continue;
}
#[cfg(test)]
if self.mixed.is_some() && analysis_source::mixed::long_analysis_word(self.text()) {
return Err(self.error("Use the short source8 analysis declaration word."));
}
self.syntax_entry()?;
let word = self.text().to_owned();
let kind = declaration_kind(&word)
.ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?;
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.record(
matches!(kind, DeclKind::Element(..) | DeclKind::Relation(..)),
Span {
start: offset,
end: offset,
},
)?;
}
graph_count += usize::from(matches!(
kind,
DeclKind::Element(..) | DeclKind::Relation(..)
@@ -1485,6 +1631,10 @@ impl Parser<'_> {
let body_start = self.current().span.end;
self.take("{")?;
let lexical_owner = open.last().map(|&owner| declarations[owner].id);
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.lexical_owners.push(open.last().copied());
}
declarations.push(Declaration {
kind,
alias,
crates/oa-core/src/language/analysis_source.rs
+1
View File
@@ -6,6 +6,7 @@ mod declarations;
mod index;
mod lex;
mod lower;
pub(super) mod mixed;
mod syntax;
mod update;
mod write;
crates/oa-core/src/language/analysis_source/mixed.rs
+394
View File
@@ -0,0 +1,394 @@
//! Private syntax-only complete-bundle witness. No model admission or edit authority.
//!
//! One scan charges original bytes and raw tokens (including trivia). Typed
//! syntax keeps its existing entry/reference/numeric charges. Ordinary syntax
//! charges one entry per project/declaration/include/description, field, and
//! value before allocation; array values already account for their list slots.
//! These private storage ceilings are independent of caller model record counts.
//! Ordinary references consume value storage, not ordered-model-step allowance.
//! Typed reference occurrences retain the caller max_step_references ceiling.
//! Only declaration graph/record counts are known here; semantic nested-model
//! records and ordinary ordered steps still require the later complete admission.
//! Metadata phase slots aggregate across all files: retained token-end indices,
//! value-end entries, projected value/field/header/declaration positions, lexical
//! owner slots and the file table. Checks do not recharge syntax or latch errors;
//! every failure must return immediately. Tokens themselves use the token bound.
//! This deliberately bounded staging policy is not calibrated public source8.
use super::{
budget::{Budget, Limits},
cursor::Cursor,
declarations, lex,
};
use crate::Id;
use crate::language::{
self, Ast, EditDeclaration, Error, Lexeme, ParseLimits, Parser, SourceUnit, Span, TokenClass,
};
use crate::source_project::{ProjectLimits, SourceBundle, SourceDiagnostic, traversal::Traversal};
use std::collections::BTreeMap;
struct ModelCounts {
limits: ProjectLimits,
graph: usize,
records: usize,
includes: usize,
}
pub(super) struct ParsedMixedSyntax<'s> {
bundle: &'s SourceBundle,
files: Vec<FileSyntax<'s>>,
budget: Budget,
}
struct FileSyntax<'s> {
path: &'s str,
source: &'s str,
tokens: Vec<lex::Token>,
ordinary: SourceUnit,
typed: Vec<declarations::Declaration>,
positions: Vec<(Id, EditDeclaration)>,
// A lexical owner is an occurrence index, not only a possibly repeated UUID.
lexical_owners: Vec<Option<usize>>,
}
// Only Parser borrows this state; the stored witness has no self-reference.
pub(in crate::language) struct State<'a> {
source: &'a str,
tokens: &'a [lex::Token],
budget: &'a mut Budget,
model: &'a mut ModelCounts,
typed: Vec<declarations::Declaration>,
pub(in crate::language) value_ends: BTreeMap<usize, usize>,
pub(in crate::language) lexical_owners: Vec<Option<usize>>,
}
fn ordinary_error(error: super::Error) -> Error {
Error {
code: "invalid_source",
offset: error.span.start,
message: error.message,
}
}
fn lexeme(token: lex::Token) -> Lexeme {
Lexeme {
class: match token.class {
lex::Class::Word => TokenClass::Identifier,
lex::Class::Number => TokenClass::Number,
lex::Class::String => TokenClass::String,
lex::Class::Punctuation => TokenClass::Punctuation,
},
span: token.span,
}
}
impl State<'_> {
pub(in crate::language) fn token(&self, at: usize) -> Option<Lexeme> {
self.tokens.get(at).copied().map(lexeme)
}
pub(in crate::language) fn entry(&mut self, at: Span) -> Result<(), Error> {
self.budget.entry(at).map_err(ordinary_error)
}
pub(in crate::language) fn record(&mut self, graph: bool, at: Span) -> Result<(), Error> {
let records = self.model.records.checked_add(1);
let graph = self.model.graph.checked_add(usize::from(graph));
let (Some(records), Some(graph)) = (records, graph) else {
return Err(Error {
code: "model_limit",
offset: at.start,
message: "Declaration count overflow.".into(),
});
};
self.model
.limits
.model
.check_counts(graph, records, 0)
.map_err(|message| Error {
code: "model_limit",
offset: at.start,
message,
})?;
self.model.records = records;
self.model.graph = graph;
Ok(())
}
pub(in crate::language) fn include(&mut self, at: Span) -> Result<(), Error> {
if self.model.includes >= self.model.limits.max_include_edges {
return Err(Error {
code: "project_limit",
offset: at.start,
message: format!(
"Include graph exceeds {} edges.",
self.model.limits.max_include_edges
),
});
}
self.model.includes += 1;
Ok(())
}
pub(in crate::language) fn ordinary_number(
&self,
at: &mut usize,
) -> Result<serde_json::Value, Error> {
let first = self.tokens[*at];
let mut end = first.span.end;
let mut after = *at + 1;
// Match the old greedy numeric alphabet only across byte-adjacent raw
// tokens. This is consumption of the shared stream, not another scan.
while let Some(next) = self.tokens.get(after) {
if next.span.start != end
|| !self.source[next.span.start..next.span.end]
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"_.+-".contains(&byte))
{
break;
}
end = next.span.end;
after += 1;
}
let value: serde_json::Number = serde_json::from_str(&self.source[first.span.start..end])
.map_err(|_| Error {
code: "invalid_source",
offset: first.span.start,
message: "Expected a byte-adjacent finite JSON decimal number.".into(),
})?;
*at = after;
Ok(serde_json::Value::Number(value))
}
pub(in crate::language) fn typed_header(&self, at: usize, root: bool) -> bool {
let Some(token) = self.tokens.get(at) else {
return false;
};
if declarations::declaration_kind(&self.source[token.span.start..token.span.end]).is_none()
{
return false;
}
// A generic field named `unit` remains an ordinary field. A declaration
// header has an alias and display-name token; only that shape is nested.
root || (self
.tokens
.get(at + 1)
.is_some_and(|t| t.class == lex::Class::Word)
&& self
.tokens
.get(at + 2)
.is_some_and(|t| t.class == lex::Class::String))
}
pub(in crate::language) fn parse_typed(
&mut self,
at: &mut usize,
owner: Option<Id>,
) -> Result<(), Error> {
// Transfer the one active position into the typed consumer and back;
// neither consumer scans, clones a stream, nor runs concurrently.
self.record(true, self.tokens[*at].span)?;
let mut cursor = Cursor::new(self.source, self.tokens, self.budget);
cursor.cursor = *at;
let result = declarations::parse(&mut cursor, 8, owner);
*at = cursor.cursor;
let declaration = result.map_err(ordinary_error)?;
let missing = declaration.header.identity.value.is_none()
|| matches!(&declaration.body, declarations::Body::Calculation { parameters, .. }
if parameters.iter().any(|p| p.identity.value.is_none()));
if missing {
return Err(Error {
code: "invalid_source",
offset: declaration.span.start,
message: "Private mixed syntax requires explicit IDs.".into(),
});
}
self.typed.push(declaration);
Ok(())
}
}
pub(in crate::language) fn long_analysis_word(word: &str) -> bool {
matches!(
word,
"oa_engineering_property"
| "sa_engineering_property"
| "oa_analysis_check"
| "sa_analysis_check"
| "oa_analysis_case"
| "sa_analysis_case"
)
}
pub(super) fn parse(
bundle: &SourceBundle,
limits: ProjectLimits,
) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
let profile = Limits::source8(ParseLimits {
// Storage is not the caller's final model record/ordered-step accounting.
max_source_bytes: limits.model.max_source_bytes,
max_step_references: limits.model.max_step_references,
..ParseLimits::UNLIMITED
});
parse_with_profile(bundle, limits, profile)
}
fn located(path: &str, source: &str, error: Error) -> Vec<SourceDiagnostic> {
vec![SourceDiagnostic {
path: path.into(),
diagnostic: language::diagnostic(source, error),
related: Vec::new(),
}]
}
fn parse_with_profile(
bundle: &SourceBundle,
limits: ProjectLimits,
profile: Limits,
) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
let entry_source = bundle.files.get(&bundle.entry).map_or("", String::as_str);
let at_entry = |error| located(&bundle.entry, entry_source, ordinary_error(error));
let maximum = Limits::source8(ParseLimits {
max_source_bytes: limits.model.max_source_bytes,
max_step_references: limits.model.max_step_references,
..ParseLimits::UNLIMITED
});
if profile.bytes > maximum.bytes
|| profile.tokens > maximum.tokens
|| profile.entries > maximum.entries
|| profile.references > maximum.references
|| profile.frames > maximum.frames
|| profile.numeric_units > maximum.numeric_units
{
return Err(at_entry(super::error(
Span::default(),
"Mixed test profile may only tighten inherited limits.",
)));
}
let mut budget = Budget::new(profile).map_err(at_entry)?;
let mut traversal = Traversal::new(bundle, limits)?;
let bytes = bundle
.files
.values()
.try_fold(0usize, |n, s| n.checked_add(s.len()))
.ok_or_else(|| at_entry(super::error(Span::default(), "Source size overflow.")))?;
budget
.index_bytes(bytes, Span::default())
.map_err(at_entry)?;
let mut metadata = bundle.files.len();
budget
.lowering_entries(metadata, Span::default())
.map_err(at_entry)?;
let mut parsed = BTreeMap::new();
let mut model = ModelCounts {
limits,
graph: 0,
records: 0,
includes: 0,
};
while let Some(path) = traversal.next()? {
let (path, source) = bundle
.files
.get_key_value(&path)
.expect("traversal checked source");
let failure = |error| located(path, source, ordinary_error(error));
let tokens = lex::tokens(source, &mut budget).map_err(failure)?;
let mut parser = Parser {
source,
tokens: Vec::new(),
cursor: 0,
value_depth: 0,
limits: limits.model,
mixed: Some(State {
source,
tokens: &tokens,
budget: &mut budget,
model: &mut model,
typed: Vec::new(),
value_ends: BTreeMap::new(),
lexical_owners: Vec::new(),
}),
};
let ordinary = parser
.unit(path == &bundle.entry, true)
.map_err(|e| located(path, source, e))?;
let state = parser.mixed.take().expect("private parser state");
drop(parser);
// Projected ordinary AST nodes/fields are bounded before constructing a
// second position representation. Count references as slots, not syntax.
let projection_slots = ordinary_position_slots(&ordinary);
metadata = metadata
.checked_add(tokens.len())
.and_then(|n| n.checked_add(state.value_ends.len()))
.and_then(|n| n.checked_add(projection_slots))
.and_then(|n| n.checked_add(state.lexical_owners.len()))
.ok_or_else(|| failure(super::error(Span::default(), "Metadata size overflow.")))?;
state
.budget
.lowering_entries(metadata, Span::default())
.map_err(failure)?;
let positions = language::edit_positions_from_tokens(
source,
&ordinary,
|at| lexeme(tokens[at]),
tokens.len(),
Some(&state.value_ends),
)
.map_err(|e| {
vec![SourceDiagnostic {
path: path.clone(),
diagnostic: *e,
related: Vec::new(),
}]
})?;
for (raw, span) in &ordinary.includes {
traversal.include(path, raw, *span)?;
}
traversal.finish_file(path.clone());
let State {
typed,
lexical_owners,
..
} = state;
parsed.insert(
path.as_str(),
FileSyntax {
path,
source,
tokens,
ordinary,
typed,
positions,
lexical_owners,
},
);
}
traversal.finish()?;
budget.active().map_err(at_entry)?;
Ok(ParsedMixedSyntax {
bundle,
files: parsed.into_values().collect(),
budget,
})
}
fn ordinary_position_slots(unit: &SourceUnit) -> usize {
// Iterative accounting avoids adding a second recursive walk before the
// already depth-bounded position projection. Each reference is one value.
let mut slots = 0usize;
let mut stack = Vec::new();
for declaration in &unit.declarations {
slots = slots.saturating_add(7); // declaration/body + at most five header roles
slots = slots.saturating_add(declaration.fields.len());
stack.extend(declaration.fields.values());
}
while let Some(value) = stack.pop() {
slots = slots.saturating_add(1);
match value {
Ast::Array(items, _) => stack.extend(items),
Ast::Object(fields, _) => {
slots = slots.saturating_add(fields.len());
stack.extend(fields.values());
}
_ => {}
}
}
slots
}
#[cfg(test)]
mod tests;
crates/oa-core/src/language/analysis_source/mixed/tests.rs
+921
View File
@@ -0,0 +1,921 @@
use super::*;
use crate::language::{EditKind, EditValue, ParseLimits, Span};
use crate::source_project::{ProjectLimits, SourceBundle};
use std::collections::BTreeMap;
const PROJECT_ID: &str = "00000000-0000-0000-0000-000000000001";
const FIRST_ID: &str = "00000000-0000-0000-0000-000000000010";
const SECOND_ID: &str = "00000000-0000-0000-0000-000000000011";
const CALC_ID: &str = "00000000-0000-0000-0000-000000000100";
const PARAM_ID: &str = "00000000-0000-0000-0000-000000000200";
fn document(body: &str) -> String {
format!("arcadia 8;\nproject p \"Project\" id \"{PROJECT_ID}\" {{\n{body}\n}}\n")
}
fn single(body: &str) -> SourceBundle {
SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([("main.arcadia".into(), document(body))]),
}
}
fn calculation(alias: &str, id: &str, expression: &str) -> String {
format!(
"calculation {alias} \"Calculation\" id \"{id}\" {{ result Number; body {expression}; }}"
)
}
fn ordinary(alias: &str, id: &str, fields: &str) -> String {
format!("oa_activity {alias} \"Activity\" id \"{id}\" {{ {fields} }}")
}
fn text(source: &str, span: Span) -> &str {
&source[span.start..span.end]
}
fn exact_span(source: &str, authored: &str) -> Span {
// The expected substring is authored here, independently of parser output.
let start = source.find(authored).expect("unique authored fixture text");
assert_eq!(source.matches(authored).count(), 1);
Span {
start,
end: start + authored.len(),
}
}
fn fields(value: &EditValue) -> &BTreeMap<String, crate::language::EditField> {
let EditKind::Record(fields) = &value.kind else {
panic!("record position")
};
fields
}
fn rejects(bundle: &SourceBundle) -> Vec<crate::source_project::SourceDiagnostic> {
let before = bundle.clone();
let errors = match parse(bundle, ProjectLimits::default()) {
Ok(_) => panic!("invalid syntax returned a complete mixed witness"),
Err(errors) => errors,
};
assert!(!errors.is_empty());
assert_eq!(bundle, &before, "failure may not stabilize or mutate input");
errors
}
#[test]
fn interleaved_unicode_crlf_has_original_typed_and_ordinary_coordinates() {
let typed = format!("calculation c \"数値\" id \"{CALC_ID}\" {{ result Number; body 1-2; }}");
let child = format!("oa_activity child \"子\" id \"{SECOND_ID}\" {{ description \"雪 @c\"; }}");
let parent = format!("oa_activity container \"親\" id \"{FIRST_ID}\" {{\r\n {child}\r\n}}");
let source = format!(
"// 雪 before every coordinate\r\narcadia 8;\r\nproject p \"港\" id \"{PROJECT_ID}\" {{\r\n description \"計画\";\r\n {parent}\r\n {typed}\r\n include \"lib.arcadia\";\r\n}}\r\n"
);
let library = ordinary("later", PARAM_ID, "description \"é\";");
let bundle = SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([
("main.arcadia".into(), source.clone()),
("lib.arcadia".into(), library),
]),
};
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
assert!(std::ptr::eq(parsed.bundle, &bundle));
assert_eq!(
parsed.files.iter().map(|f| f.path).collect::<Vec<_>>(),
["lib.arcadia", "main.arcadia"]
);
let file = &parsed.files[1];
assert!(std::ptr::eq(
file.source.as_ptr(),
bundle.files["main.arcadia"].as_ptr()
));
assert_eq!(file.source.len(), source.len());
assert_eq!(file.ordinary.declarations.len(), 2);
assert_eq!(file.typed.len(), 1);
assert_eq!(file.positions.len(), 2);
assert_eq!(file.lexical_owners, [None, Some(0)]);
assert_eq!(
file.ordinary.declarations[0].span,
exact_span(&source, &parent)
);
assert_eq!(
file.ordinary.declarations[1].span,
exact_span(&source, &child)
);
assert_eq!(file.positions[1].1.span, exact_span(&source, &child));
assert_eq!(file.typed[0].span, exact_span(&source, &typed));
let description = &fields(&file.positions[1].1.fields)["description"];
assert_eq!(
description.span,
exact_span(&source, "description \"雪 @c\";")
);
assert_eq!(description.value.span, exact_span(&source, "\"雪 @c\""));
assert_eq!(
file.ordinary.includes[0].1,
exact_span(&source, "include \"lib.arcadia\";")
);
assert_eq!(
text(
&source,
file.ordinary.header.as_ref().unwrap().description.unwrap()
),
"description \"計画\";"
);
assert_eq!(
parsed.budget.usage.bytes,
bundle.files.values().map(String::len).sum::<usize>()
);
assert!(
file.tokens
.windows(2)
.all(|pair| pair[0].span.end <= pair[1].span.start)
);
assert!(
file.tokens
.iter()
.all(|token| source.is_char_boundary(token.span.start)
&& source.is_char_boundary(token.span.end))
);
let declarations::Body::Calculation { body, .. } = &file.typed[0].body else {
panic!("calculation")
};
assert_eq!(text(&source, body.value_span), "1-2");
let super::super::Binding::Expression(expression) = &body.value else {
panic!("expression")
};
assert!(matches!(
&expression.kind,
super::super::ExpressionKind::Binary {
op: crate::analysis::BinaryOp::Subtract,
..
}
));
}
#[test]
fn duplicate_ordinary_ids_keep_occurrences_and_scoped_parameter_ids_remain_distinct() {
let child_a = ordinary("child_a", SECOND_ID, "");
let child_b = ordinary("child_b", SECOND_ID, "");
let parent_a = ordinary("parent_a", FIRST_ID, &child_a);
let parent_b = ordinary("parent_b", FIRST_ID, &child_b);
let calc = |alias: &str, id: &str| {
format!(
"calculation {alias} \"Scoped\" id \"{id}\" {{ input p id \"{PARAM_ID}\" : Number; result Number; body $p; }}"
)
};
let bundle = single(&format!(
"{parent_a}\n{}\n{parent_b}\n{}",
calc("f", CALC_ID),
calc("g", PARAM_ID)
));
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
let file = &parsed.files[0];
assert_eq!(
file.ordinary
.declarations
.iter()
.map(|d| d.alias.as_str())
.collect::<Vec<_>>(),
["parent_a", "child_a", "parent_b", "child_b"]
);
assert_eq!(file.lexical_owners, [None, Some(0), None, Some(2)]);
assert_eq!(
file.positions.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
[FIRST_ID, SECOND_ID, FIRST_ID, SECOND_ID].map(|id| Id::parse_str(id).unwrap())
);
for (index, authored) in [&parent_a, &child_a, &parent_b, &child_b]
.into_iter()
.enumerate()
{
assert_eq!(
file.positions[index].1.span,
exact_span(file.source, authored)
);
}
assert_eq!(file.typed.len(), 2);
for declaration in &file.typed {
let declarations::Body::Calculation {
parameters, body, ..
} = &declaration.body
else {
panic!("calculation")
};
assert_eq!(parameters.len(), 1);
assert_eq!(
parameters[0].identity.value,
Some(Id::parse_str(PARAM_ID).unwrap())
);
assert_eq!(parameters[0].name.name, "p");
assert_eq!(text(file.source, body.value_span), "$p");
}
// Syntax deliberately does not establish global identity uniqueness.
assert_eq!(
file.typed[1].header.identity.value,
Some(Id::parse_str(PARAM_ID).unwrap())
);
}
#[test]
fn diamond_includes_return_one_canonical_file_table_and_scan_dependency_once() {
let bundle = SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([
(
"main.arcadia".into(),
document("include \"root/z.arcadia\"; include \"root/a.arcadia\";"),
),
(
"root/z.arcadia".into(),
"include \"../shared.arcadia\";".into(),
),
(
"root/a.arcadia".into(),
"include \"../shared.arcadia\";".into(),
),
(
"shared.arcadia".into(),
calculation("shared", CALC_ID, "1e-2"),
),
]),
};
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
assert_eq!(
parsed.files.iter().map(|f| f.path).collect::<Vec<_>>(),
[
"main.arcadia",
"root/a.arcadia",
"root/z.arcadia",
"shared.arcadia"
]
);
assert_eq!(parsed.files.iter().map(|f| f.typed.len()).sum::<usize>(), 1);
assert_eq!(
parsed
.files
.iter()
.map(|f| f.ordinary.includes.len())
.sum::<usize>(),
4
);
assert_eq!(
parsed.budget.usage.bytes,
bundle.files.values().map(String::len).sum::<usize>()
);
assert_eq!(
parsed
.files
.iter()
.filter(|f| f.ordinary.header.is_some())
.count(),
1
);
assert!(parsed.files[0].ordinary.header.is_some());
let limits = ProjectLimits {
max_files: 4,
max_include_depth: 2,
max_include_edges: 4,
..ProjectLimits::default()
};
assert!(parse(&bundle, limits).is_ok());
for limits in [
ProjectLimits {
max_files: 3,
..limits
},
ProjectLimits {
max_include_depth: 1,
..limits
},
ProjectLimits {
max_include_edges: 3,
..limits
},
] {
assert!(parse(&bundle, limits).is_err());
}
}
#[test]
fn whole_bundle_rejects_missing_unreachable_cyclic_and_noncanonical_files() {
let mut missing = single("include \"missing.arcadia\";");
rejects(&missing);
missing.entry = "absent.arcadia".into();
rejects(&missing);
let mut extra = single("");
extra.files.insert(
"orphan.arcadia".into(),
"// untouched but unreachable".into(),
);
rejects(&extra);
let mut cycle = single("include \"child.arcadia\";");
cycle
.files
.insert("child.arcadia".into(), "include \"main.arcadia\";".into());
rejects(&cycle);
let mut collision = single("include \"MAIN.arcadia\";");
collision.files.insert("MAIN.arcadia".into(), "".into());
rejects(&collision);
for key in [
"./child.arcadia",
"x/../child.arcadia",
"dir\\child.arcadia",
"/child.arcadia",
] {
let mut bundle = single("");
bundle.files.insert(key.into(), "".into());
rejects(&bundle);
}
for include in [
"../outside.arcadia",
"/absolute.arcadia",
"x\\child.arcadia",
] {
// JSON quoting deliberately makes the backslash an actual path byte.
rejects(&single(&format!(
"include {};",
serde_json::to_string(include).unwrap()
)));
}
}
#[test]
fn ordinary_signed_numbers_keep_complete_nested_value_and_field_ends() {
let body = ordinary(
"a",
FIRST_ID,
"weight -12.5; nested { value -1e-2; items [-3, { tail -4.5; }]; }; parent @target;",
);
let bundle = single(&body);
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
let file = &parsed.files[0];
let top = fields(&file.positions[0].1.fields);
for (key, value, field) in [
("weight", "-12.5", "weight -12.5;"),
(
"nested",
"{ value -1e-2; items [-3, { tail -4.5; }]; }",
"nested { value -1e-2; items [-3, { tail -4.5; }]; };",
),
("parent", "@target", "parent @target;"),
] {
assert_eq!(top[key].value.span, exact_span(file.source, value));
assert_eq!(top[key].span, exact_span(file.source, field));
assert_eq!(
file.ordinary.declarations[0].field_spans[key],
exact_span(file.source, value)
);
}
let nested = fields(&top["nested"].value);
assert_eq!(nested["value"].value.span, exact_span(file.source, "-1e-2"));
assert_eq!(
nested["value"].span,
exact_span(file.source, "value -1e-2;")
);
assert_eq!(
nested["items"].span,
exact_span(file.source, "items [-3, { tail -4.5; }];")
);
let EditKind::List(items) = &nested["items"].value.kind else {
panic!("list")
};
assert_eq!(items.len(), 2);
assert_eq!(items[0].span, exact_span(file.source, "-3"));
assert_eq!(items[1].span, exact_span(file.source, "{ tail -4.5; }"));
assert_eq!(
fields(&items[1])["tail"].span,
exact_span(file.source, "tail -4.5;")
);
assert_eq!(
fields(&items[1])["tail"].value.span,
exact_span(file.source, "-4.5")
);
// Equal-length version substitution retains all byte coordinates. These
// comparisons supplement, rather than generate, the authored oracles above.
let legacy = file.source.replacen("arcadia 8;", "arcadia 7;", 1);
let legacy_positions =
crate::language::edit_declarations(&legacy, true, ParseLimits::UNLIMITED).unwrap();
let old = fields(&legacy_positions[0].1.fields);
assert_eq!(old["weight"].value.span, top["weight"].value.span);
assert_eq!(old["nested"].span, top["nested"].span);
assert_eq!(
fields(&old["nested"].value)["items"].span,
nested["items"].span
);
}
#[test]
fn ordinary_numeric_adjacency_and_malformed_trivia_match_legacy_refusals() {
for value in [
"- 12.5",
"- // comment\n12.5",
"+1",
".5",
"1e",
"1_0",
"0x1",
"1-2",
"1e9999",
"@ /*not supported*/ target",
] {
for field in [
format!("weight {value};"),
format!("nested {{ items [{value}]; }};"),
] {
let bundle = single(&ordinary("a", FIRST_ID, &field));
rejects(&bundle);
let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1);
assert!(
crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_err(),
"legacy unexpectedly accepts {value}"
);
}
}
for value in ["-12.5", "-1e-2", "1e-2", "0", "-0", "1E+2"] {
let bundle = single(&ordinary("a", FIRST_ID, &format!("weight {value};")));
assert!(
parse(&bundle, ProjectLimits::default()).is_ok(),
"mixed refuses {value}"
);
let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1);
assert!(
crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_ok(),
"legacy refuses {value}"
);
}
}
#[test]
fn typed_words_are_contextual_but_typed_headers_cannot_nest_in_ordinary_bodies() {
let bundle = single(&format!(
"{}\n{}",
ordinary(
"unit",
FIRST_ID,
"unit 1; calculation \"word\"; quantity_kind none;"
),
calculation("calculation", CALC_ID, "1")
));
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
let file = &parsed.files[0];
assert_eq!(file.ordinary.declarations[0].alias, "unit");
assert_eq!(file.typed[0].header.alias.name, "calculation");
assert_eq!(
fields(&file.positions[0].1.fields)
.keys()
.map(String::as_str)
.collect::<Vec<_>>(),
["calculation", "quantity_kind", "unit"]
);
rejects(&single(&ordinary(
"a",
FIRST_ID,
&calculation("c", CALC_ID, "1"),
)));
for word in [
"oa_engineering_property",
"sa_engineering_property",
"oa_analysis_check",
"sa_analysis_check",
"oa_analysis_case",
"sa_analysis_case",
] {
let forbidden = format!("{word} a \"Long derived kind\" id \"{FIRST_ID}\" {{}}");
rejects(&single(&forbidden));
rejects(&single(&ordinary("owner", SECOND_ID, &forbidden)));
}
}
#[test]
fn every_identity_class_requires_authored_ids_without_mutating_the_bundle() {
let project_missing = SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([(
"main.arcadia".into(),
"arcadia 8; project p \"Project\" {}".into(),
)]),
};
rejects(&project_missing);
for body in [
"oa_activity a \"Missing\" {}".to_owned(),
ordinary("owner", FIRST_ID, "oa_activity child \"Missing\" {}"),
"calculation c \"Missing\" { result Number; body 1; }".to_owned(),
format!(
"calculation c \"Missing parameter\" id \"{CALC_ID}\" {{ input p : Number; result Number; body $p; }}"
),
"quantity_kind q \"Missing\" { dimension {}; algebra Linear; }".to_owned(),
"unit u \"Missing\" { kind @q; scale 1; offset 0; symbol \"u\"; }".to_owned(),
] {
rejects(&single(&body));
}
let mut included = single("include \"lib.arcadia\";");
included
.files
.insert("lib.arcadia".into(), "oa_activity a \"Missing\" {}".into());
rejects(&included);
}
#[test]
fn ordinary_value_depth_boundary_remains_64_in_both_parser_views() {
// A scalar consumes the innermost level: 63 containing lists + scalar = 64.
for (lists, accepted) in [(63, true), (64, false)] {
let value = format!("{}1{}", "[".repeat(lists), "]".repeat(lists));
let bundle = single(&ordinary("a", FIRST_ID, &format!("values {value};")));
assert_eq!(parse(&bundle, ProjectLimits::default()).is_ok(), accepted);
let legacy = bundle.files["main.arcadia"].replacen("arcadia 8;", "arcadia 7;", 1);
assert_eq!(
crate::language::parse_unit(&legacy, true, ParseLimits::UNLIMITED).is_ok(),
accepted
);
}
}
#[test]
fn exact_shared_profile_limits_include_nonlatching_metadata_phase_failures() {
// Independent hand count: 10 significant tokens and 5 intervening spaces.
// No declaration positions, value-end entries, or lexical owners exist.
// The phase table therefore needs 1 file slot + 10 token-end slots = 11.
let source = format!("arcadia 8;project p \"P\" id \"{PROJECT_ID}\"{{}}");
let bundle = SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([("main.arcadia".into(), source.clone())]),
};
let limits = ProjectLimits {
model: ParseLimits {
max_records: 1,
max_step_references: 0,
max_source_bytes: source.len(),
..ParseLimits::UNLIMITED
},
..ProjectLimits::default()
};
let profile = Limits {
bytes: source.len(),
tokens: 15,
entries: 11,
references: 0,
numeric_units: 0,
..Limits::source8(ParseLimits::UNLIMITED)
};
let parsed = parse_with_profile(&bundle, limits, profile).unwrap();
assert_eq!(parsed.files.len(), 1);
assert_eq!(parsed.files[0].tokens.len(), 10);
assert_eq!(parsed.budget.usage.bytes, source.len());
assert_eq!(parsed.budget.usage.tokens, 15);
assert_eq!(parsed.budget.usage.entries, 1);
assert_eq!(parsed.budget.usage.references, 0);
assert_eq!(parsed.budget.numeric_units(), 0);
for (profile, message) in [
(
Limits {
entries: 10,
..profile
},
"Source8 lowering entry limit exceeded.",
),
(
Limits {
tokens: 14,
..profile
},
"Source8 tokens limit exceeded.",
),
(
Limits {
bytes: source.len() - 1,
..profile
},
"Source8 index/output byte limit exceeded.",
),
] {
let errors = match parse_with_profile(&bundle, limits, profile) {
Ok(_) => panic!("exhausted phase returned a complete witness"),
Err(errors) => errors,
};
assert!(
errors
.iter()
.any(|error| error.diagnostic.message == message)
);
assert_eq!(bundle.files["main.arcadia"], source);
}
// A caller source-byte limit is inherited even when a test profile would
// otherwise fall within the candidate hard ceiling.
let tighter = ProjectLimits {
model: ParseLimits {
max_source_bytes: source.len() - 1,
..limits.model
},
..limits
};
assert!(parse_with_profile(&bundle, tighter, profile).is_err());
}
#[test]
fn ordinary_model_records_and_parent_reference_are_not_syntax_or_step_limits() {
let bundle = single(&ordinary("a", FIRST_ID, "parent @target;"));
let limits = ProjectLimits {
model: ParseLimits {
max_records: 2,
max_step_references: 0,
..ParseLimits::UNLIMITED
},
..ProjectLimits::default()
};
let parsed = parse(&bundle, limits).unwrap();
assert_eq!(parsed.files[0].ordinary.declarations.len(), 1);
// Project, ordinary declaration, field, and reference value: four storage
// entries, while only project + declaration count as two model records.
assert_eq!(parsed.budget.usage.entries, 4);
assert_eq!(parsed.budget.usage.references, 0);
assert_eq!(
text(
parsed.files[0].source,
fields(&parsed.files[0].positions[0].1.fields)["parent"]
.value
.span
),
"@target"
);
assert!(
parse(
&bundle,
ProjectLimits {
model: ParseLimits {
max_records: 1,
..limits.model
},
..limits
}
)
.is_err()
);
}
#[test]
fn typed_numeric_work_remains_in_the_same_inherited_profile() {
let bundle = single(&calculation("c", CALC_ID, "1e-2"));
let limits = ProjectLimits::default();
let parsed = parse(&bundle, limits).unwrap();
assert!(parsed.budget.numeric_units() > 0);
let profile = Limits {
numeric_units: 0,
..Limits::source8(ParseLimits::UNLIMITED)
};
let before = bundle.clone();
assert!(parse_with_profile(&bundle, limits, profile).is_err());
assert_eq!(bundle, before);
}
#[test]
fn all_short_typed_roots_share_original_buffers_without_public_activation() {
use crate::Layer;
use declarations::DeclarationKind as K;
let rows = [
(
"quantity_kind",
"q",
"dimension {}; algebra Linear;",
K::QuantityKind,
),
(
"unit",
"u",
"kind @q; scale ratio(1, 10); offset 0; symbol \"u\";",
K::Unit,
),
(
"calculation",
"c",
"result Number; body 123456789012345678901234567890123456789;",
K::Calculation,
),
(
"oa_property",
"op",
"subject @later; type Number; base unbound;",
K::Property(Layer::Oa),
),
(
"sa_property",
"sp",
"subject @later; type Number; base unbound;",
K::Property(Layer::Sa),
),
(
"oa_check",
"oc",
"subject none; predicate true;",
K::Check(Layer::Oa),
),
(
"sa_check",
"sc",
"subject none; predicate true;",
K::Check(Layer::Sa),
),
(
"oa_analysis",
"case_oa",
"subject none; base ModelBaseline; overrides {};",
K::Case(Layer::Oa),
),
(
"sa_analysis",
"case_sa",
"subject none; base ModelBaseline; overrides {};",
K::Case(Layer::Sa),
),
];
let declarations: Vec<_> = rows
.iter()
.enumerate()
.map(|(n, (word, alias, body, _))| {
format!(
"{word} {alias} \"Typed\" id \"00000000-0000-0000-0000-{:012}\" {{ {body} }}",
n + 1000
)
})
.collect();
let bundle = single(&format!(
"{}\n{}",
declarations.join("\n"),
ordinary("later", FIRST_ID, "")
));
let parsed = parse(&bundle, ProjectLimits::default()).unwrap();
let file = &parsed.files[0];
assert_eq!(file.typed.len(), 9);
for ((actual, (_, alias, _, kind)), authored) in file.typed.iter().zip(rows).zip(&declarations)
{
assert_eq!(actual.header.kind, kind);
assert_eq!(actual.header.alias.name, alias);
assert_eq!(actual.span, exact_span(file.source, authored));
}
let declarations::Body::Calculation { body, .. } = &file.typed[2].body else {
panic!("calculation")
};
let super::super::Binding::Expression(expression) = &body.value else {
panic!("expression")
};
let super::super::ExpressionKind::Number(value) = &expression.kind else {
panic!("exact number")
};
assert_eq!(
value.canonical_literal(),
"123456789012345678901234567890123456789"
);
// The checked private mode does not change any public source8 entrypoint.
assert!(crate::language::parse_unit(file.source, true, ParseLimits::UNLIMITED).is_err());
assert!(crate::source_project::compile_bundle(&bundle, ProjectLimits::default()).is_err());
let old = SourceBundle {
entry: bundle.entry.clone(),
files: BTreeMap::from([(
bundle.entry.clone(),
file.source.replacen("arcadia 8", "arcadia 7", 1),
)]),
};
rejects(&old);
let mut fragment_header = single("include \"fragment.arcadia\";");
fragment_header
.files
.insert("fragment.arcadia".into(), document(""));
rejects(&fragment_header);
}
#[test]
fn mixed_typed_reference_limit_is_inherited_without_charging_ordinary_parent_refs() {
let bundle = single(&format!(
"{}\n{}",
ordinary("a", FIRST_ID, "parent @unresolved;"),
calculation("c", CALC_ID, "@unresolved")
));
let limits = ProjectLimits {
model: ParseLimits {
max_step_references: 1,
..ParseLimits::UNLIMITED
},
..ProjectLimits::default()
};
let parsed = parse(&bundle, limits).unwrap();
assert_eq!(parsed.budget.usage.references, 1);
assert!(
parse(
&bundle,
ProjectLimits {
model: ParseLimits {
max_step_references: 0,
..limits.model
},
..limits
}
)
.is_err()
);
}
#[test]
fn caller_declaration_counts_are_aggregate_and_checked_before_typed_header_allocation() {
let mut bundle = single(&format!(
"{}\ninclude \"later.arcadia\";",
ordinary("early", FIRST_ID, "")
));
bundle
.files
.insert("later.arcadia".into(), calculation("later", CALC_ID, "1"));
let limits = ProjectLimits {
model: ParseLimits {
max_records: 3,
max_graph_objects: 2,
..ParseLimits::UNLIMITED
},
..ProjectLimits::default()
};
assert!(parse(&bundle, limits).is_ok());
// The third declaration is deliberately missing its header fields. A tighter
// aggregate record ceiling must refuse at its kind before parsing/allocating
// the alias, name, ID, or body; a fresh per-file counter would miss it.
bundle
.files
.insert("later.arcadia".into(), "calculation".into());
for model in [
ParseLimits {
max_records: 2,
..limits.model
},
ParseLimits {
max_graph_objects: 1,
..limits.model
},
] {
let errors = match parse(&bundle, ProjectLimits { model, ..limits }) {
Ok(_) => panic!("record limit returned a witness"),
Err(errors) => errors,
};
assert_eq!(errors[0].path, "later.arcadia");
assert_eq!(errors[0].diagnostic.code, "model_limit");
assert_eq!(errors[0].diagnostic.span.start, 0);
}
let includes = single("include true;");
let errors = match parse(
&includes,
ProjectLimits {
max_include_edges: 0,
..limits
},
) {
Ok(_) => panic!("include limit returned a witness"),
Err(errors) => errors,
};
assert_eq!(errors[0].diagnostic.code, "project_limit");
assert!(errors[0].diagnostic.message.contains("Include graph"));
}
#[test]
fn mixed_deep_typed_success_and_rejection_cleanup_on_128k_stack() {
std::thread::Builder::new()
.name("mixed-syntax-128k".into())
.stack_size(128 * 1024)
.spawn(|| {
for (wrappers, accepted) in [(127, true), (128, false)] {
let expression = format!("{}1{}", "abs(".repeat(wrappers), ")".repeat(wrappers));
let bundle = single(&format!(
"{}\n{}\n{}",
ordinary("before", FIRST_ID, ""),
calculation("c", CALC_ID, &expression),
ordinary("after", SECOND_ID, "")
));
match parse(&bundle, ProjectLimits::default()) {
Ok(witness) => {
assert!(accepted);
let declarations::Body::Calculation { body, .. } =
&witness.files[0].typed[0].body
else {
panic!("calculation")
};
let super::super::Binding::Expression(expr) = &body.value else {
panic!("expression")
};
assert_eq!(expr.depth, 128);
drop(witness);
}
Err(errors) => {
assert!(!accepted);
assert!(
errors
.iter()
.any(|e| e.diagnostic.message.contains("depth"))
);
}
}
}
assert!(
parse(
&single(&calculation("fresh", CALC_ID, "1")),
ProjectLimits::default()
)
.is_ok()
);
})
.unwrap()
.join()
.unwrap();
}
crates/oa-core/src/source_project.rs
+7 -147
View File
@@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet};
mod source_migration;
#[path = "source_project/source_structure.rs"]
mod source_structure;
pub(crate) mod traversal;
/// A portable slash-separated path, checked at every bundle boundary.
pub type LogicalPath = String;
@@ -304,71 +305,7 @@ fn compile_once(
bundle: &SourceBundle,
limits: ProjectLimits,
) -> Result<(CompiledSourceProject, Edits), Vec<SourceDiagnostic>> {
let fail = |code, message| {
vec![problem(
bundle,
&bundle.entry,
Span::default(),
code,
message,
)]
};
if bundle.entry.contains('/') {
return Err(fail(
"invalid_path",
"The entry file must be at the source project root.".into(),
));
}
if bundle.files.len() > limits.max_files {
return Err(fail(
"project_limit",
format!("Source project exceeds {} files.", limits.max_files),
));
}
let bytes = bundle
.files
.values()
.fold(0usize, |sum, text| sum.saturating_add(text.len()));
if bytes > limits.model.max_source_bytes {
return Err(fail(
"source_limit",
format!(
"Combined source exceeds {} bytes.",
limits.model.max_source_bytes
),
));
}
let mut folded = BTreeMap::new();
for path in std::iter::once(&bundle.entry).chain(bundle.files.keys()) {
if resolve_include("", path).as_ref() != Ok(path) {
return Err(fail(
"invalid_path",
format!("Non-canonical source path '{path}'."),
));
}
if let Some(previous) = folded.insert(path.to_lowercase(), path)
&& previous != path
{
return Err(fail(
"duplicate_path",
format!("Case-colliding source paths '{previous}' and '{path}'."),
));
}
}
if !bundle.files.contains_key(&bundle.entry) {
return Err(fail(
"missing_include",
"The entry file is missing from the source bundle.".into(),
));
}
enum Visit {
Enter(LogicalPath, usize, Option<SourceLocation>),
Leave,
}
let mut stack = vec![Visit::Enter(bundle.entry.clone(), 0, None)];
let mut active: Vec<LogicalPath> = Vec::new();
let mut visited = BTreeSet::new();
let mut traversal = traversal::Traversal::new(bundle, limits)?;
let mut declarations = Vec::new();
let mut declaration_paths = Vec::new();
let mut aliases = BTreeMap::new();
@@ -381,51 +318,8 @@ fn compile_once(
let mut includes = Vec::new();
let mut insertions: Edits = BTreeMap::new();
let mut graph_count = 0usize;
while let Some(visit) = stack.pop() {
let Visit::Enter(path, depth, origin) = visit else {
active.pop();
continue;
};
let at = origin.unwrap_or(SourceLocation {
path: path.clone(),
span: Span::default(),
});
if let Some(start) = active.iter().position(|file| file == &path) {
let chain = active[start..]
.iter()
.chain(std::iter::once(&path))
.cloned()
.collect::<Vec<_>>()
.join(" -> ");
return Err(vec![problem(
bundle,
&at.path,
at.span,
"include_cycle",
format!("Include cycle: {chain}."),
)]);
}
if depth > limits.max_include_depth {
return Err(vec![problem(
bundle,
&at.path,
at.span,
"project_limit",
format!("Include depth exceeds {}.", limits.max_include_depth),
)]);
}
if !visited.insert(path.clone()) {
continue;
}
let Some(source) = bundle.files.get(&path) else {
return Err(vec![problem(
bundle,
&at.path,
at.span,
"missing_include",
format!("Included file '{path}' is missing."),
)]);
};
while let Some(path) = traversal.next()? {
let source = &bundle.files[&path];
let mut unit = language::parse_unit(source, path == bundle.entry, limits.model).map_err(
|diagnostic| {
vec![SourceDiagnostic {
@@ -513,46 +407,12 @@ fn compile_once(
declarations.push(declaration);
declaration_paths.push(path.clone());
}
let mut children = Vec::new();
for (raw, span) in unit.includes {
let target = resolve_include(&path, &raw)
.map_err(|message| vec![problem(bundle, &path, span, "invalid_path", message)])?;
let location = SourceLocation {
path: path.clone(),
span,
};
includes.push(IncludeLocation {
location: location.clone(),
target: target.clone(),
});
if includes.len() > limits.max_include_edges {
return Err(vec![problem(
bundle,
&path,
span,
"project_limit",
format!("Include graph exceeds {} edges.", limits.max_include_edges),
)]);
}
children.push(Visit::Enter(
target,
depth.saturating_add(1),
Some(location),
));
includes.push(traversal.include(&path, &raw, span)?);
}
active.push(path);
stack.push(Visit::Leave);
stack.extend(children.into_iter().rev());
}
if let Some(path) = bundle.files.keys().find(|path| !visited.contains(*path)) {
return Err(vec![problem(
bundle,
path,
Span::default(),
"unreachable_file",
"Supplied source file is not reachable from the entry file.",
)]);
traversal.finish_file(path);
}
traversal.finish()?;
let header = project_header.expect("validated entry header");
let index = SourceIndex {
declarations: declaration_locations,
crates/oa-core/src/source_project/traversal.rs
+258
View File
@@ -0,0 +1,258 @@
//! Shared include traversal policy for ordinary compilation and private syntax.
use super::*;
enum Visit {
Enter(LogicalPath, usize, Option<SourceLocation>),
Leave,
}
pub(crate) struct Traversal<'a> {
bundle: &'a SourceBundle,
limits: ProjectLimits,
stack: Vec<Visit>,
active: Vec<LogicalPath>,
visited: BTreeSet<LogicalPath>,
children: Vec<Visit>,
depth: usize,
edges: usize,
}
impl<'a> Traversal<'a> {
pub(crate) fn new(
bundle: &'a SourceBundle,
limits: ProjectLimits,
) -> Result<Self, Vec<SourceDiagnostic>> {
let fail = |code, message| {
vec![problem(
bundle,
&bundle.entry,
Span::default(),
code,
message,
)]
};
if bundle.entry.contains('/') {
return Err(fail(
"invalid_path",
"The entry file must be at the source project root.".into(),
));
}
if bundle.files.len() > limits.max_files {
return Err(fail(
"project_limit",
format!("Source project exceeds {} files.", limits.max_files),
));
}
let bytes = bundle
.files
.values()
.fold(0usize, |sum, text| sum.saturating_add(text.len()));
if bytes > limits.model.max_source_bytes {
return Err(fail(
"source_limit",
format!(
"Combined source exceeds {} bytes.",
limits.model.max_source_bytes
),
));
}
let mut folded = BTreeMap::new();
for path in std::iter::once(&bundle.entry).chain(bundle.files.keys()) {
if resolve_include("", path).as_ref() != Ok(path) {
return Err(fail(
"invalid_path",
format!("Non-canonical source path '{path}'."),
));
}
if let Some(previous) = folded.insert(path.to_lowercase(), path)
&& previous != path
{
return Err(fail(
"duplicate_path",
format!("Case-colliding source paths '{previous}' and '{path}'."),
));
}
}
if !bundle.files.contains_key(&bundle.entry) {
return Err(fail(
"missing_include",
"The entry file is missing from the source bundle.".into(),
));
}
Ok(Self {
bundle,
limits,
stack: vec![Visit::Enter(bundle.entry.clone(), 0, None)],
active: Vec::new(),
visited: BTreeSet::new(),
children: Vec::new(),
depth: 0,
edges: 0,
})
}
pub(crate) fn next(&mut self) -> Result<Option<LogicalPath>, Vec<SourceDiagnostic>> {
while let Some(visit) = self.stack.pop() {
let Visit::Enter(path, depth, origin) = visit else {
self.active.pop();
continue;
};
let at = origin.unwrap_or(SourceLocation {
path: path.clone(),
span: Span::default(),
});
if let Some(start) = self.active.iter().position(|file| file == &path) {
let chain = self.active[start..]
.iter()
.chain(std::iter::once(&path))
.cloned()
.collect::<Vec<_>>()
.join(" -> ");
return Err(vec![problem(
self.bundle,
&at.path,
at.span,
"include_cycle",
format!("Include cycle: {chain}."),
)]);
}
if depth > self.limits.max_include_depth {
return Err(vec![problem(
self.bundle,
&at.path,
at.span,
"project_limit",
format!("Include depth exceeds {}.", self.limits.max_include_depth),
)]);
}
if !self.visited.insert(path.clone()) {
continue;
}
if !self.bundle.files.contains_key(&path) {
return Err(vec![problem(
self.bundle,
&at.path,
at.span,
"missing_include",
format!("Included file '{path}' is missing."),
)]);
}
self.depth = depth;
return Ok(Some(path));
}
Ok(None)
}
pub(crate) fn include(
&mut self,
path: &str,
raw: &str,
span: Span,
) -> Result<IncludeLocation, Vec<SourceDiagnostic>> {
let target = resolve_include(path, raw)
.map_err(|message| vec![problem(self.bundle, path, span, "invalid_path", message)])?;
let location = SourceLocation {
path: path.into(),
span,
};
if self.edges >= self.limits.max_include_edges {
return Err(vec![problem(
self.bundle,
path,
span,
"project_limit",
format!(
"Include graph exceeds {} edges.",
self.limits.max_include_edges
),
)]);
}
self.edges += 1;
self.children.push(Visit::Enter(
target.clone(),
self.depth.saturating_add(1),
Some(location.clone()),
));
Ok(IncludeLocation { location, target })
}
pub(crate) fn finish_file(&mut self, path: LogicalPath) {
self.active.push(path);
self.stack.push(Visit::Leave);
self.stack.extend(self.children.drain(..).rev());
}
pub(crate) fn finish(self) -> Result<(), Vec<SourceDiagnostic>> {
if let Some(path) = self
.bundle
.files
.keys()
.find(|path| !self.visited.contains(*path))
{
return Err(vec![problem(
self.bundle,
path,
Span::default(),
"unreachable_file",
"Supplied source file is not reachable from the entry file.",
)]);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ordinary_compile_once_keeps_include_order_and_cycle_source_location() {
let mut bundle = SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([
("main.arcadia".into(), "arcadia 7; project p \"P\" id \"00000000-0000-0000-0000-000000000001\" { include \"dir/left.arcadia\"; include \"dir/right.arcadia\"; }".into()),
("dir/left.arcadia".into(), "include \"../shared.arcadia\";".into()),
("dir/right.arcadia".into(), "include \"../shared.arcadia\";".into()),
("shared.arcadia".into(), "// one shared ordinary fragment\n".into()),
]),
};
let original = bundle.clone();
// This is the existing initial ordinary pass, before schema migration's
// separate output/reparse. It makes no public source8 acceptance claim.
let (compiled, insertions) =
super::super::compile_once(&bundle, ProjectLimits::default()).unwrap();
assert!(insertions.is_empty());
assert_eq!(compiled.bundle, original);
assert_eq!(compiled.index.declarations.len(), 1);
assert_eq!(
compiled
.index
.includes
.iter()
.map(|i| (i.location.path.as_str(), i.target.as_str()))
.collect::<Vec<_>>(),
[
("main.arcadia", "dir/left.arcadia"),
("main.arcadia", "dir/right.arcadia"),
("dir/left.arcadia", "shared.arcadia"),
("dir/right.arcadia", "shared.arcadia"),
]
);
let cycle = "// 雪\r\ninclude \"main.arcadia\";";
bundle.files.insert("shared.arcadia".into(), cycle.into());
let before = bundle.clone();
let errors = super::super::compile_once(&bundle, ProjectLimits::default()).unwrap_err();
assert_eq!(errors[0].diagnostic.code, "include_cycle");
assert_eq!(errors[0].path, "shared.arcadia");
assert_eq!(errors[0].diagnostic.line, 2);
assert_eq!(
&cycle[errors[0].diagnostic.span.start..errors[0].diagnostic.span.end],
"include \"main.arcadia\";"
);
assert_eq!(
errors[0].diagnostic.message,
"Include cycle: main.arcadia -> dir/left.arcadia -> shared.arcadia -> main.arcadia."
);
assert_eq!(bundle, before);
}
}