Compare commits

...
14 changed files with 1505 additions and 41 deletions
+153 -22
View File
@@ -1018,6 +1018,28 @@ fn edit_positions_from_tokens(
token: impl Fn(usize) -> Lexeme,
count: usize,
value_ends: Option<&BTreeMap<usize, usize>>,
) -> Result<Vec<(Id, EditDeclaration)>, Box<Diagnostic>> {
edit_positions_from_tokens_with_text(source, unit, token, count, value_ends, |_, _| Ok(()))
}
fn edit_positions_from_tokens_with_text(
source: &str,
unit: &SourceUnit,
token: impl Fn(usize) -> Lexeme,
count: usize,
value_ends: Option<&BTreeMap<usize, usize>>,
text: impl FnMut(usize, Span) -> Result<(), Box<Diagnostic>>,
) -> Result<Vec<(Id, EditDeclaration)>, Box<Diagnostic>> {
edit_positions_from_tokens_with_producers(source, unit, token, count, value_ends, None, text)
}
fn edit_positions_from_tokens_with_producers(
source: &str,
unit: &SourceUnit,
token: impl Fn(usize) -> Lexeme,
count: usize,
value_ends: Option<&BTreeMap<usize, usize>>,
mut quota: Option<usize>,
text: impl FnMut(usize, Span) -> Result<(), Box<Diagnostic>>,
) -> Result<Vec<(Id, EditDeclaration)>, Box<Diagnostic>> {
let mut ends: Vec<_> = (0..count).map(|at| token(at).span.end).collect();
let mut stack = Vec::new();
@@ -1029,12 +1051,32 @@ fn edit_positions_from_tokens(
_ => {}
}
}
struct Positions<'a, F> {
struct Positions<'a, F, H> {
token_at: F,
ends: Vec<usize>,
value_ends: Option<&'a BTreeMap<usize, usize>>,
text: H,
source: &'a str,
quota: &'a mut Option<usize>,
}
impl<F: Fn(usize) -> Lexeme> Positions<'_, F> {
impl<F: Fn(usize) -> Lexeme, H: FnMut(usize, Span) -> Result<(), Box<Diagnostic>>>
Positions<'_, F, H>
{
fn consume(&mut self, slots: usize, offset: usize) -> Result<(), Box<Diagnostic>> {
if let Some(remaining) = self.quota {
*remaining = remaining.checked_sub(slots).ok_or_else(|| {
Box::new(diagnostic(
self.source,
Error {
code: "invalid_source",
offset,
message: "Source8 projection quota underflow.".into(),
},
))
})?;
}
Ok(())
}
fn token(&self, start: usize) -> usize {
let mut low = 0;
let mut high = self.ends.len();
@@ -1048,24 +1090,30 @@ fn edit_positions_from_tokens(
}
low
}
fn fields(&self, fields: &BTreeMap<String, Ast>) -> BTreeMap<String, EditField> {
fn fields(
&mut self,
fields: &BTreeMap<String, Ast>,
) -> Result<BTreeMap<String, EditField>, Box<Diagnostic>> {
fields
.iter()
.map(|(key, ast)| {
let value = self.value(ast);
self.consume(1, ast.offset())?;
let value = self.value(ast)?;
let start = (self.token_at)(self.token(value.span.start) - 1).span.start;
let end = (self.token_at)(self.token(value.span.end)).span.end;
(
(self.text)(key.len(), Span { start, end })?;
Ok((
key.clone(),
EditField {
span: Span { start, end },
value,
},
)
))
})
.collect()
}
fn value(&self, ast: &Ast) -> EditValue {
fn value(&mut self, ast: &Ast) -> Result<EditValue, Box<Diagnostic>> {
self.consume(1, ast.offset())?;
let index = self.token(ast.offset());
let span = Span {
start: ast.offset(),
@@ -1080,23 +1128,33 @@ fn edit_positions_from_tokens(
}),
};
let kind = match ast {
Ast::Object(fields, _) => EditKind::Record(self.fields(fields)),
Ast::Array(items, _) => {
EditKind::List(items.iter().map(|item| self.value(item)).collect())
Ast::Object(fields, _) => EditKind::Record(self.fields(fields)?),
Ast::Array(items, _) => EditKind::List(
items
.iter()
.map(|item| self.value(item))
.collect::<Result<_, _>>()?,
),
Ast::Reference(alias, _) => {
(self.text)(alias.len(), span)?;
EditKind::Reference(alias.clone())
}
Ast::Reference(alias, _) => EditKind::Reference(alias.clone()),
Ast::HistoricalUuid(id, _) => EditKind::HistoricalUuid(*id),
_ => EditKind::Atom,
};
EditValue { span, kind }
Ok(EditValue { span, kind })
}
}
let positions = Positions {
let mut positions = Positions {
token_at: &token,
ends,
value_ends,
text,
source,
quota: &mut quota,
};
unit.declarations
let result = unit
.declarations
.iter()
.map(|declaration| {
if declaration.id_insertion.is_some() {
@@ -1109,6 +1167,7 @@ fn edit_positions_from_tokens(
},
)));
}
positions.consume(7, declaration.span.start)?;
let roles: &[&str] = if matches!(declaration.kind, DeclKind::Node) {
&["kind", "alias", "id_keyword", "id"]
} else {
@@ -1132,12 +1191,23 @@ fn edit_positions_from_tokens(
start: declaration.body.start - 1,
end: declaration.body.end + 1,
},
kind: EditKind::Record(positions.fields(&declaration.fields)),
kind: EditKind::Record(positions.fields(&declaration.fields)?),
},
},
))
})
.collect()
.collect::<Result<Vec<_>, _>>()?;
if quota.is_some_and(|remaining| remaining != 0) {
return Err(Box::new(diagnostic(
source,
Error {
code: "invalid_source",
offset: 0,
message: "Source8 projection quota leftover.".into(),
},
)));
}
Ok(result)
}
struct Parser<'a> {
source: &'a str,
@@ -1196,11 +1266,21 @@ impl Parser<'_> {
if self.current().class != TokenClass::String {
return Err(self.error("Expected a double-quoted string."));
}
#[cfg(test)]
let span = self.current().span;
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
let raw = &self.source[span.start..span.end];
let value = mixed.quoted_text(raw, span)?;
self.cursor += 1;
return Ok(value);
}
let value = serde_json::from_str(self.text()).unwrap();
self.cursor += 1;
Ok(value)
}
fn identifier(&mut self) -> Result<String, Error> {
self.own_text(self.text().len(), self.current().span)?;
let value = self.text().to_owned();
if !crate::valid_alias(&value) {
return Err(self.error(std::format!("Invalid or reserved alias '{value}'.")));
@@ -1230,11 +1310,32 @@ impl Parser<'_> {
message: "Expected UUID in 8-4-4-4-12 hexadecimal form.".into(),
});
}
Id::parse_str(&value).map_err(|_| Error {
let id = Id::parse_str(&value).map_err(|_| Error {
code: "invalid_source",
offset,
message: "Invalid UUID identity.".into(),
})
})?;
let bytes = value.len();
drop(value);
self.release_text(bytes);
Ok(id)
}
fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> {
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.own_text(bytes, span)?;
}
#[cfg(not(test))]
let _ = (bytes, span);
Ok(())
}
fn release_text(&mut self, bytes: usize) {
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.release_text(bytes);
}
#[cfg(not(test))]
let _ = bytes;
}
fn fields(&mut self) -> Result<BTreeMap<String, Ast>, Error> {
self.take("{")?;
@@ -1242,6 +1343,13 @@ impl Parser<'_> {
while self.text() != "}" {
let offset = self.current().span.start;
let (key, value, _) = self.field()?;
self.own_text(
key.len(),
Span {
start: offset,
end: offset,
},
)?;
if fields.insert(key.clone(), value).is_some() {
return Err(Error {
code: "invalid_source",
@@ -1249,12 +1357,16 @@ impl Parser<'_> {
message: std::format!("Duplicate field '{key}'."),
});
}
let bytes = key.len();
drop(key);
self.release_text(bytes);
}
self.take("}")?;
Ok(fields)
}
fn field(&mut self) -> Result<(String, Ast, Span), Error> {
self.syntax_entry()?;
self.own_text(self.text().len(), self.current().span)?;
let key = self.text().to_owned();
if !matches!(
self.current().class,
@@ -1271,6 +1383,10 @@ impl Parser<'_> {
.span
.end;
self.take(";")?;
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.projection(1, Span { start, end })?;
}
Ok((key, value, Span { start, end }))
}
@@ -1304,7 +1420,12 @@ impl Parser<'_> {
.span
.end;
if let Some(mixed) = &mut self.mixed {
mixed.value_ends.insert(ast.offset(), end);
let span = Span {
start: ast.offset(),
end,
};
mixed.value_end(span)?;
mixed.projection(1, span)?;
}
}
result
@@ -1328,6 +1449,9 @@ impl Parser<'_> {
offset,
message: "Expected a UUID after uuid.".into(),
})?;
let bytes = value.len();
drop(value);
self.release_text(bytes);
Ok(Ast::HistoricalUuid(id, offset))
}
"@" => {
@@ -1368,6 +1492,7 @@ impl Parser<'_> {
Ok(Ast::Literal(value, offset))
}
TokenClass::Identifier | TokenClass::Keyword => {
self.own_text(self.text().len(), self.current().span)?;
let value = self.text().to_owned();
self.cursor += 1;
Ok(Ast::Word(value, offset))
@@ -1514,6 +1639,7 @@ impl Parser<'_> {
}
} else {
let (key, value, span) = self.field()?;
self.own_text(key.len(), span)?;
let declaration = &mut declarations[owner];
if declaration.fields.insert(key.clone(), value).is_some() {
return Err(Error {
@@ -1584,8 +1710,8 @@ impl Parser<'_> {
return Err(self.error("Use the short source8 analysis declaration word."));
}
self.syntax_entry()?;
let word = self.text().to_owned();
let kind = declaration_kind(&word)
let word = self.current().text(self.source);
let kind = declaration_kind(word)
.ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?;
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
@@ -1626,7 +1752,12 @@ impl Parser<'_> {
let lexical_owner = open.last().map(|&owner| declarations[owner].id);
#[cfg(test)]
if let Some(mixed) = &mut self.mixed {
mixed.lexical_owners.push(open.last().copied());
let span = Span {
start: offset,
end: body_start,
};
mixed.lexical_owner(open.last().copied(), span)?;
mixed.projection(7, span)?;
}
declarations.push(Declaration {
kind,
@@ -38,6 +38,17 @@ pub struct Budget {
pub usage: Usage,
numeric: WorkMeter,
failed: Option<Error>,
text: Option<ProducerText>,
entries: Option<ProducerEntries>,
}
struct ProducerEntries {
carried: usize,
metadata: usize,
}
struct ProducerText {
carried: usize,
owned: usize,
temporary: usize,
}
impl Budget {
pub fn new(limits: Limits) -> Result<Self, Error> {
@@ -62,8 +73,153 @@ impl Budget {
.map_err(|e| error(Span::default(), e.to_string()))?,
),
failed: None,
text: None,
entries: None,
})
}
pub(super) fn enable_entry_producers(
&mut self,
carried: usize,
files: usize,
) -> Result<(), Error> {
assert!(self.entries.is_none());
self.check_producer_entries(carried, files, 0, Span::default())?;
self.entries = Some(ProducerEntries {
carried,
metadata: files,
});
Ok(())
}
fn check_producer_entries(
&mut self,
carried: usize,
metadata: usize,
delta: usize,
span: Span,
) -> Result<(), Error> {
self.active()?;
let total = carried
.checked_add(self.usage.entries)
.and_then(|n| n.checked_add(metadata))
.and_then(|n| n.checked_add(delta));
let message = match total {
None => Some("Source8 producer entry size overflow."),
Some(n) if n > self.limits.entries => Some("Source8 lowering entry limit exceeded."),
_ => None,
};
if let Some(message) = message {
let failure = error(span, message);
self.failed = Some(failure.clone());
return Err(failure);
}
Ok(())
}
pub(super) fn producer_entries(&self) -> Option<usize> {
self.entries.as_ref().map(|e| e.metadata)
}
pub(super) fn reserve_metadata(&mut self, delta: usize, span: Span) -> Result<(), Error> {
if let Some(e) = &self.entries {
self.check_producer_entries(e.carried, e.metadata, delta, span)?;
self.entries.as_mut().expect("enabled entries").metadata += delta;
}
Ok(())
}
pub(super) fn producer_entry_overflow(&mut self, span: Span) -> Error {
if let Some(failure) = &self.failed {
return failure.clone();
}
let failure = error(span, "Source8 producer entry size overflow.");
self.failed = Some(failure.clone());
failure
}
pub(super) fn enable_text_producers(&mut self, carried: usize) -> Result<(), Error> {
self.index_bytes(carried, Span::default())?;
assert!(self.text.is_none());
self.text = Some(ProducerText {
carried,
owned: 0,
temporary: 0,
});
Ok(())
}
pub(super) fn producer_text(&self) -> Option<(usize, usize)> {
self.text.as_ref().map(|t| (t.owned, t.temporary))
}
fn reserve_text(&mut self, bytes: usize, temporary: bool, span: Span) -> Result<(), Error> {
let Some(t) = &self.text else {
return Ok(());
};
let total = t
.carried
.checked_add(t.owned)
.and_then(|n| n.checked_add(t.temporary))
.and_then(|n| n.checked_add(bytes))
.ok_or_else(|| error(span, "Source8 producer text size overflow."))?;
self.index_bytes(total, span)?;
let t = self.text.as_mut().expect("enabled producer text");
if temporary {
t.temporary += bytes;
} else {
t.owned += bytes;
}
Ok(())
}
pub(super) fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> {
self.reserve_text(bytes, false, span)
}
pub(super) fn temporary_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> {
self.reserve_text(bytes, true, span)
}
pub(super) fn release_text(&mut self, bytes: usize) {
if let Some(t) = &mut self.text {
t.owned = t.owned.checked_sub(bytes).expect("owned text reservation");
}
}
pub(super) fn release_temporary_text(&mut self, bytes: usize) {
if let Some(t) = &mut self.text {
t.temporary = t
.temporary
.checked_sub(bytes)
.expect("temporary text reservation");
}
}
pub(super) fn quoted_text(&mut self, raw: &str, span: Span) -> Result<String, Error> {
if self.text.is_none() {
return serde_json::from_str(raw).map_err(|_| error(span, "Invalid quoted string."));
}
let payload = raw
.len()
.checked_sub(2)
.ok_or_else(|| error(span, "Invalid quoted text extent."))?;
self.temporary_text(payload, span)?;
let result = serde_json::from_str::<String>(raw);
let value = match result {
Ok(value) => value,
Err(_) => {
self.release_temporary_text(payload);
return Err(error(span, "Invalid quoted string."));
}
};
if value.len() > payload {
drop(value);
self.release_temporary_text(payload);
return Err(error(span, "Decoded text exceeds its payload reservation."));
}
// Keep the reservation continuous while moving decoded bytes into T.
// D <= P and the original checked aggregate bound prove both sums fit.
let t = self.text.as_mut().expect("enabled producer text");
let temporary = t
.temporary
.checked_sub(payload)
.expect("quoted text reservation");
let owned = t
.owned
.checked_add(value.len())
.expect("bounded decoded transfer");
t.temporary = temporary;
t.owned = owned;
Ok(value)
}
/// Immutable exact policy snapshot for private deliberate-snapshot continuation.
pub(super) fn limits(&self) -> Limits {
self.limits
@@ -105,6 +261,9 @@ impl Budget {
span,
"syntax entries",
)?;
if let Some(e) = &self.entries {
self.check_producer_entries(e.carried, e.metadata, 1, span)?;
}
self.usage.entries += 1;
Ok(())
}
@@ -74,6 +74,7 @@ impl Cursor<'_, '_> {
{
return Err(error(span, "Expected a valid reference or parameter name."));
}
self.budget.own_text(self.text().len(), span)?;
let name = self.text().to_owned();
self.cursor += 1;
Ok(Reference { name, span })
@@ -87,8 +88,8 @@ impl Cursor<'_, '_> {
{
return Err(error(span, "Expected a quoted string."));
}
let value =
serde_json::from_str(self.text()).map_err(|_| error(span, "Invalid quoted string."))?;
let raw = &self.source[span.start..span.end];
let value = self.budget.quoted_text(raw, span)?;
self.cursor += 1;
Ok(value)
}
@@ -426,6 +426,9 @@ impl Cursor<'_, '_> {
));
}
let id = Id::parse_str(&spelling).map_err(|_| error(literal, "Invalid UUID identity."))?;
let bytes = spelling.len();
drop(spelling);
self.budget.release_text(bytes);
Ok(IdentitySyntax {
value: Some(id),
keyword: Some(keyword),
@@ -1,5 +1,5 @@
//! Source8 significant-token adapter over the shared raw scanner.
use super::super::raw_tokens::{Class as RawClass, Scanner};
use super::super::raw_tokens::{Class as RawClass, Scanner, StringEvent};
use super::{Budget, Error, Span, error};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -33,7 +33,20 @@ pub fn tokens(source: &str, budget: &mut Budget) -> Result<Vec<Token>, Error> {
start: offset,
end: offset,
})?;
let token = scanner.next().expect("source remains");
let token = if budget.producer_text().is_some() {
scanner.next_with_strings(|event| {
match event {
StringEvent::Reserve { span, payload } => {
budget.temporary_text(payload, span)?
}
StringEvent::Released { payload } => budget.release_temporary_text(payload),
}
Ok::<(), Error>(())
})?
} else {
scanner.next()
}
.expect("source remains");
offset = token.span.end;
let class = match token.class {
RawClass::Word => Class::Word,
@@ -68,8 +68,11 @@ pub(in crate::language) struct State<'a> {
budget: &'a mut Budget,
model: &'a mut ModelCounts,
typed: Vec<declarations::Declaration>,
pub(in crate::language) value_ends: BTreeMap<usize, usize>,
pub(in crate::language) lexical_owners: Vec<Option<usize>>,
value_ends: BTreeMap<usize, usize>,
lexical_owners: Vec<Option<usize>>,
pending_projection: usize,
reserved_values: usize,
reserved_owners: usize,
}
fn ordinary_error(error: super::Error) -> Error {
@@ -93,6 +96,55 @@ fn lexeme(token: lex::Token) -> Lexeme {
}
impl State<'_> {
pub(in crate::language) fn projection(
&mut self,
slots: usize,
span: Span,
) -> Result<(), Error> {
if self.budget.producer_entries().is_some() {
self.pending_projection = self
.pending_projection
.checked_add(slots)
.ok_or_else(|| ordinary_error(self.budget.producer_entry_overflow(span)))?;
}
Ok(())
}
pub(in crate::language) fn value_end(&mut self, span: Span) -> Result<(), Error> {
let new_key = !self.value_ends.contains_key(&span.start);
self.budget
.reserve_metadata(usize::from(new_key), span)
.map_err(ordinary_error)?;
if new_key {
self.reserved_values += 1;
}
self.value_ends.insert(span.start, span.end);
Ok(())
}
pub(in crate::language) fn lexical_owner(
&mut self,
owner: Option<usize>,
span: Span,
) -> Result<(), Error> {
self.budget
.reserve_metadata(1, span)
.map_err(ordinary_error)?;
self.lexical_owners.push(owner);
self.reserved_owners += 1;
Ok(())
}
pub(in crate::language) fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> {
self.budget.own_text(bytes, span).map_err(ordinary_error)
}
pub(in crate::language) fn release_text(&mut self, bytes: usize) {
self.budget.release_text(bytes);
}
pub(in crate::language) fn quoted_text(
&mut self,
raw: &str,
span: Span,
) -> Result<String, Error> {
self.budget.quoted_text(raw, span).map_err(ordinary_error)
}
pub(in crate::language) fn token(&self, at: usize) -> Option<Lexeme> {
self.tokens.get(at).copied().map(lexeme)
}
@@ -250,7 +302,7 @@ fn parse_with_profile(
limits: ProjectLimits,
profile: Limits,
) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
parse_impl(bundle, limits, profile, None)
parse_impl(bundle, limits, profile, None, false, false)
}
fn parse_impl<'s>(
@@ -258,6 +310,8 @@ fn parse_impl<'s>(
limits: ProjectLimits,
profile: Limits,
carried: Option<carried::CarriedGuard<'s>>,
text_producers: bool,
entry_producers: bool,
) -> Result<ParsedMixedSyntax<'s>, Vec<SourceDiagnostic>> {
let entry_source = bundle.files.get(&bundle.entry).map_or("", String::as_str);
let at_entry = |error| located(&bundle.entry, entry_source, ordinary_error(error));
@@ -279,6 +333,14 @@ fn parse_impl<'s>(
)));
}
let mut budget = Budget::new(profile).map_err(at_entry)?;
if text_producers {
let input = carried
.as_ref()
.expect("text mode requires carried authority");
budget
.enable_text_producers(input.slots().text)
.map_err(at_entry)?;
}
if let Some(input) = &carried {
input.check(&budget, 0, 0).map_err(at_entry)?;
}
@@ -291,7 +353,14 @@ fn parse_impl<'s>(
.index_bytes(bytes, Span::default())
.map_err(at_entry)?;
let mut metadata = bundle.files.len();
if let Some(input) = &carried {
if entry_producers {
let input = carried
.as_ref()
.expect("entry mode requires carried authority");
budget
.enable_entry_producers(input.slots().entries, metadata)
.map_err(at_entry)?;
} else if let Some(input) = &carried {
input.check(&budget, 0, metadata).map_err(at_entry)?;
} else {
budget
@@ -315,6 +384,7 @@ fn parse_impl<'s>(
.expect("traversal checked source");
let failure = |error| located(path, source, ordinary_error(error));
let tokens = lex::tokens(source, &mut budget).map_err(failure)?;
let metadata_before = budget.producer_entries();
let mut parser = Parser {
source,
tokens: Vec::new(),
@@ -329,6 +399,9 @@ fn parse_impl<'s>(
typed: Vec::new(),
value_ends: BTreeMap::new(),
lexical_owners: Vec::new(),
pending_projection: 0,
reserved_values: 0,
reserved_owners: 0,
}),
};
let ordinary = parser
@@ -338,7 +411,19 @@ fn parse_impl<'s>(
drop(parser);
// Projected ordinary AST nodes/fields are bounded before constructing a
// second position representation. Count references as slots, not syntax.
let projection_slots = ordinary_position_slots(&ordinary);
let projection_slots = if entry_producers {
let slots = tokens
.len()
.checked_add(state.pending_projection)
.ok_or_else(|| failure(state.budget.producer_entry_overflow(Span::default())))?;
state
.budget
.reserve_metadata(slots, Span::default())
.map_err(failure)?;
state.pending_projection
} else {
ordinary_position_slots(&ordinary)
};
metadata = metadata
.checked_add(tokens.len())
.and_then(|n| n.checked_add(state.value_ends.len()))
@@ -355,12 +440,19 @@ fn parse_impl<'s>(
.lowering_entries(metadata, Span::default())
.map_err(failure)?;
}
let positions = language::edit_positions_from_tokens(
let positions = language::edit_positions_from_tokens_with_producers(
source,
&ordinary,
|at| lexeme(tokens[at]),
tokens.len(),
Some(&state.value_ends),
entry_producers.then_some(projection_slots),
|bytes, span| {
state
.budget
.own_text(bytes, span)
.map_err(|e| Box::new(language::diagnostic(source, ordinary_error(e))))
},
)
.map_err(|e| {
vec![SourceDiagnostic {
@@ -369,6 +461,23 @@ fn parse_impl<'s>(
related: Vec::new(),
}]
})?;
if let Some(before) = metadata_before {
let expected = tokens
.len()
.checked_add(state.value_ends.len())
.and_then(|n| n.checked_add(state.lexical_owners.len()))
.and_then(|n| n.checked_add(projection_slots))
.and_then(|n| n.checked_add(before));
if state.reserved_values != state.value_ends.len()
|| state.reserved_owners != state.lexical_owners.len()
|| expected != state.budget.producer_entries()
{
return Err(failure(super::error(
Span::default(),
"Source8 producer entry inventory mismatch.",
)));
}
}
for (raw, span) in &ordinary.includes {
traversal.include(path, raw, *span)?;
}
@@ -393,6 +502,15 @@ fn parse_impl<'s>(
}
traversal.finish()?;
budget.active().map_err(at_entry)?;
if budget.producer_entries().is_some_and(|m| m != metadata) {
return Err(at_entry(super::error(
Span::default(),
"Source8 producer entry inventory mismatch.",
)));
}
if let Some((_, temporary)) = budget.producer_text() {
assert_eq!(temporary, 0, "completed syntax has no temporary text");
}
if let Some(input) = &carried {
input
.check(&budget, budget.usage.entries, metadata)
@@ -85,7 +85,46 @@ impl CarriedGuard<'_> {
/// Input already binds the actual bundle; callers cannot supply a replacement.
fn parse_carried(input: CarriedInput<'_>) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
let (bundle, policy, guard) = input.into_parts();
parse_impl(bundle, policy.project_limits, policy.limits, Some(guard))
parse_impl(
bundle,
policy.project_limits,
policy.limits,
Some(guard),
false,
false,
)
}
/// Explicit stronger fixture route; the actual owner supplies all authority.
#[cfg(test)]
fn parse_text_producers(
input: CarriedInput<'_>,
) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
let (bundle, policy, guard) = input.into_parts();
parse_impl(
bundle,
policy.project_limits,
policy.limits,
Some(guard),
true,
false,
)
}
/// Explicit retained text and entry producer fixture route.
#[cfg(test)]
fn parse_entry_producers(
input: CarriedInput<'_>,
) -> Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>> {
let (bundle, policy, guard) = input.into_parts();
parse_impl(
bundle,
policy.project_limits,
policy.limits,
Some(guard),
true,
true,
)
}
#[cfg(test)]
@@ -139,3 +178,9 @@ impl<'s> CarriedInput<'s> {
#[cfg(test)]
mod tests;
#[cfg(test)]
mod text_producer;
#[cfg(test)]
mod entry_producer;
@@ -0,0 +1,445 @@
//! Literal entry oracles, independently authored before the producer implementation.
use super::*;
const LIMIT: &str = "Source8 lowering entry limit exceeded.";
fn doc(body: &str) -> String {
format!(
"arcadia 8;\nproject p \"Project\" id \"00000000-0000-0000-0000-000000000001\" {{\n{body}\n}}\n"
)
}
fn bundle(body: &str) -> SourceBundle {
SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([("main.arcadia".into(), doc(body))]),
}
}
fn chain() -> SourceBundle {
let mut b = bundle("include \"part.arcadia\";");
b.files.insert(
"part.arcadia".into(),
"include \"last.arcadia\";\r\n".into(),
);
b.files.insert("last.arcadia".into(), "//é\r\n".into());
b
}
fn held(bundle: SourceBundle, entries: usize) -> RetainedSnapshot {
RetainedSnapshot::fixture(
bundle,
SnapshotPolicy {
limits: Limits {
entries,
bytes: 4096,
..Limits::source8(ParseLimits::UNLIMITED)
},
project_limits: ProjectLimits::default(),
},
)
.unwrap()
}
fn failed(result: Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>>) -> SourceDiagnostic {
let errors = match result {
Ok(_) => panic!("unexpected syntax success"),
Err(e) => e,
};
assert_eq!(errors.len(), 1);
errors.into_iter().next().unwrap()
}
fn diagnostic(d: &SourceDiagnostic, path: &str, start: usize, end: usize) {
assert_eq!(d.path, path);
assert_eq!(d.diagnostic.code, "invalid_source");
assert_eq!(d.diagnostic.message, LIMIT);
assert_eq!(d.diagnostic.span, Span { start, end });
assert!(d.related.is_empty());
}
#[test]
fn real_owner_syntax_and_resolver_boundaries_are_separate() {
for (b, c, u, m, syntax_cap, resolve_cap, refused_path) in [
(bundle(""), 1, 1, 11, 13, 17, "main.arcadia"),
(chain(), 3, 3, 19, 25, 29, "part.arcadia"),
] {
let owner = held(b.clone(), syntax_cap);
let syntax = parse_entry_producers(owner.input()).unwrap();
assert_eq!(owner.retained.entries, c);
assert_eq!(syntax.budget.usage.entries, u);
assert_eq!(syntax.budget.producer_entries(), Some(m));
assert_eq!(syntax.metadata, m);
assert_eq!(
project::text_producer_inventory(&syntax).unwrap().entries,
c + u + m
);
drop(syntax);
let low = held(b.clone(), syntax_cap - 1);
diagnostic(
&failed(parse_entry_producers(low.input())),
refused_path,
0,
1,
);
for cap in [resolve_cap, resolve_cap - 1] {
let owner = held(b.clone(), cap);
let syntax = parse_entry_producers(owner.input()).unwrap();
let usage = syntax.budget.usage;
let numeric = syntax.budget.numeric_units();
match project::resolve(syntax) {
Ok(result) => {
assert_eq!(cap, resolve_cap);
assert_eq!(result.phase_state().1.entries, c + u + m);
assert_eq!(result.phase_state().0.budget.usage, usage);
assert_eq!(result.phase_state().0.budget.numeric_units(), numeric);
let text =
project::resolve(parse_text_producers(owner.input()).unwrap()).unwrap();
assert_eq!(result.project(), text.project());
assert_eq!(result.warnings(), text.warnings());
assert_eq!(result.phase_state().1, text.phase_state().1);
}
Err(failure) => {
assert_eq!(cap, resolve_cap - 1);
assert_eq!(failure.diagnostics.len(), 1);
diagnostic(&failure.diagnostics[0], "main.arcadia", 11, 73);
}
}
assert_eq!(owner.bundle, b);
}
}
}
#[test]
fn owner_and_value_refuse_at_the_original_producer() {
for (body, cap, offset, complete) in [
(
"oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { }",
4,
75,
29,
),
(
"oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { description \"\"; }",
7,
149,
37,
),
] {
let owner = held(bundle(body), cap);
diagnostic(
&failed(parse_entry_producers(owner.input())),
"main.arcadia",
offset,
offset + 1,
);
let owner = held(bundle(body), complete);
let syntax = parse_entry_producers(owner.input()).unwrap();
assert_eq!(
owner.retained.entries + syntax.budget.usage.entries + syntax.metadata,
complete
);
// The independent inventory walker has its own existing scratch peak.
let roomy = held(bundle(body), 4096);
let syntax = parse_entry_producers(roomy.input()).unwrap();
assert_eq!(
project::text_producer_inventory(&syntax).unwrap().entries,
complete
);
}
}
const NESTED: &str = "arcadia 8;\nproject p \"Project\" id \"00000000-0000-0000-0000-000000000001\" {\noa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { payload { items [@p, uuid \"00000000-0000-0000-0000-000000000001\", { flag true; }]; };\noa_activity b \"B\" id \"00000000-0000-0000-0000-000000000003\" { }\n}\nnode n id \"00000000-0000-0000-0000-000000000004\" { }\n}\n";
fn nested() -> SourceBundle {
SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([("main.arcadia".into(), NESTED.into())]),
}
}
#[test]
fn nested_inventory_and_prepaid_projection_quota() {
assert_eq!(NESTED.len(), 344);
let owner = held(nested(), 103);
let syntax = parse_entry_producers(owner.input()).unwrap();
assert_eq!(syntax.budget.usage.entries, 13);
assert_eq!(syntax.budget.producer_entries(), Some(89));
assert_eq!(syntax.metadata, 89);
assert_eq!(
owner.retained.entries + syntax.budget.usage.entries + syntax.metadata,
103
);
let roomy = held(nested(), 4096);
let inventory_syntax = parse_entry_producers(roomy.input()).unwrap();
assert_eq!(
project::text_producer_inventory(&inventory_syntax)
.unwrap()
.entries,
103
);
let file = &syntax.files[0];
assert_eq!(file.tokens.len(), 49);
assert_eq!(file.lexical_owners, [None, Some(0), None]);
assert_eq!(file.positions.len(), 3);
assert_eq!(file.positions[0].1.header.len(), 5);
assert_eq!(file.positions[2].1.header.len(), 4);
let language::EditKind::Record(fields) = &file.positions[0].1.fields.kind else {
panic!("body")
};
let payload = &fields["payload"].value;
assert_eq!(
payload.span,
Span {
start: 145,
end: 221
}
);
let language::EditKind::Record(fields) = &payload.kind else {
panic!("payload")
};
let items = &fields["items"].value;
assert_eq!(
items.span,
Span {
start: 153,
end: 218
}
);
let language::EditKind::List(items) = &items.kind else {
panic!("items")
};
assert_eq!(
items.iter().map(|v| v.span).collect::<Vec<_>>(),
[
Span {
start: 154,
end: 156
},
Span {
start: 158,
end: 201
},
Span {
start: 203,
end: 217
}
]
);
let language::EditKind::Record(fields) = &items[2].kind else {
panic!("member")
};
assert_eq!(
fields["flag"].value.span,
Span {
start: 210,
end: 214
}
);
// Injected prepaid quota errors, not valid alternate source inventories.
for (quota, message) in [
(29, "Source8 projection quota underflow."),
(31, "Source8 projection quota leftover."),
] {
let error = match language::edit_positions_from_tokens_with_producers(
file.source,
&file.ordinary,
|at| lexeme(file.tokens[at]),
file.tokens.len(),
None,
Some(quota),
|_, _| Ok(()),
) {
Err(error) => error,
Ok(_) => panic!("injected projection quota must refuse"),
};
assert_eq!(error.message, message);
}
assert_eq!(syntax.budget.producer_entries(), Some(89));
let low = held(nested(), 102);
diagnostic(
&failed(parse_entry_producers(low.input())),
"main.arcadia",
0,
1,
);
}
fn make_state<'a>(budget: &'a mut Budget, model: &'a mut ModelCounts) -> State<'a> {
State {
source: "",
tokens: &[],
budget,
model,
typed: Vec::new(),
value_ends: BTreeMap::new(),
lexical_owners: Vec::new(),
pending_projection: 0,
reserved_values: 0,
reserved_owners: 0,
}
}
fn internal_budget(cap: usize) -> Budget {
Budget::new(Limits {
entries: cap,
..Limits::source8(ParseLimits::UNLIMITED)
})
.unwrap()
}
#[test]
fn internal_reserve_and_mutate_helpers_leave_collections_unchanged() {
// Scalar fault fixtures exercise atomic mutation; these do not invent real owners.
let mut model = ModelCounts {
limits: ProjectLimits::default(),
graph: 0,
records: 0,
includes: 0,
};
let mut budget = internal_budget(3);
budget.enable_entry_producers(1, 1).unwrap();
let mut state = make_state(&mut budget, &mut model);
state.value_end(Span { start: 10, end: 11 }).unwrap();
state.value_end(Span { start: 10, end: 12 }).unwrap();
assert_eq!(state.value_ends, BTreeMap::from([(10, 12)]));
assert_eq!(state.reserved_values, 1);
assert_eq!(state.budget.producer_entries(), Some(2));
let error = state.value_end(Span { start: 20, end: 21 }).unwrap_err();
assert_eq!(error.message, LIMIT);
assert_eq!(error.offset, 20);
let repeated = state.value_end(Span { start: 10, end: 99 }).unwrap_err();
assert_eq!(repeated.offset, 20);
assert_eq!(repeated.message, LIMIT);
assert_eq!(state.value_ends, BTreeMap::from([(10, 12)]));
assert_eq!(state.budget.producer_entries(), Some(2));
let error = state
.lexical_owner(Some(0), Span { start: 30, end: 31 })
.unwrap_err();
assert_eq!(error.offset, 20); // Original latched refusal survives a later producer.
assert!(state.lexical_owners.is_empty());
assert_eq!(state.reserved_owners, 0);
let mut budget = internal_budget(3);
budget.enable_entry_producers(1, 1).unwrap();
let mut state = make_state(&mut budget, &mut model);
state.lexical_owner(None, Span::default()).unwrap();
assert!(
state
.lexical_owner(None, Span { start: 40, end: 41 })
.is_err()
);
assert_eq!(state.lexical_owners, [None]);
assert_eq!(state.reserved_owners, 1);
assert_eq!(state.budget.producer_entries(), Some(2));
}
#[test]
fn internal_overflow_latch_and_syntax_work_precedence() {
let at = Span { start: 9, end: 10 };
let mut budget = internal_budget(2);
budget.enable_entry_producers(1, 1).unwrap();
let e = budget.entry(at).unwrap_err();
assert_eq!(e.message, LIMIT);
assert_eq!(budget.producer_entry_overflow(Span::default()).span, at);
assert_eq!(budget.active().unwrap_err().message, LIMIT);
assert_eq!(budget.usage.entries, 0);
assert_eq!(budget.producer_entries(), Some(1));
assert_eq!(
budget
.reserve_metadata(0, Span::default())
.unwrap_err()
.span,
at
);
let mut budget = internal_budget(0);
budget.enable_entry_producers(0, 0).unwrap();
assert_eq!(
budget.entry(at).unwrap_err().message,
"Source8 syntax entries limit exceeded."
);
let mut budget = internal_budget(100);
budget.enable_entry_producers(1, 1).unwrap();
assert_eq!(
budget.reserve_metadata(usize::MAX, at).unwrap_err().message,
"Source8 producer entry size overflow."
);
assert_eq!(budget.producer_entries(), Some(1));
assert_eq!(budget.active().unwrap_err().span, at);
let mut budget = internal_budget(100);
assert!(budget.enable_entry_producers(usize::MAX, 1).is_err());
assert_eq!(budget.producer_entries(), None);
assert!(budget.active().is_err());
let mut budget = internal_budget(100);
budget.enable_entry_producers(0, 0).unwrap();
let mut model = ModelCounts {
limits: ProjectLimits::default(),
graph: 0,
records: 0,
includes: 0,
};
let mut state = make_state(&mut budget, &mut model);
state.pending_projection = usize::MAX;
assert_eq!(
state.projection(1, at).unwrap_err().message,
"Source8 producer entry size overflow."
);
assert_eq!(state.pending_projection, usize::MAX);
assert!(state.budget.active().is_err());
}
#[test]
fn explicit_zero_and_actual_owner_preserve_complete_oa_sa_and_old_routes() {
for (work, property) in [
("oa_activity", "oa_property"),
("sa_function", "sa_property"),
] {
let body = format!(
"calculation c \"C\" id \"00000000-0000-0000-0000-000000000003\" {{ input x id \"00000000-0000-0000-0000-000000000004\" : Number; result Number; body $x; }}\n{property} v \"V\" id \"00000000-0000-0000-0000-000000000002\" {{ subject @w; type Number; base @c(x: 1); }}\n{work} w \"W\" id \"00000000-0000-0000-0000-000000000005\" {{}}"
);
let original = bundle(&body);
let owner = held(original.clone(), 4096);
for carried in [false, true] {
let seed = parse_text_producers(owner.input()).unwrap();
let input = if carried {
owner.input()
} else {
CarriedInput::borrowed(&owner.bundle, &seed)
};
let syntax = parse_entry_producers(input).unwrap();
let input = if carried {
owner.input()
} else {
CarriedInput::borrowed(&owner.bundle, &seed)
};
let prior = parse_text_producers(input).unwrap();
assert_eq!(syntax.budget.usage, prior.budget.usage);
assert_eq!(syntax.budget.numeric_units(), prior.budget.numeric_units());
assert_eq!(syntax.metadata, prior.metadata);
assert_eq!(syntax.budget.producer_text(), prior.budget.producer_text());
assert_eq!(syntax.budget.producer_entries(), Some(syntax.metadata));
let result = project::resolve(syntax).unwrap();
let prior = project::resolve(prior).unwrap();
assert_eq!(result.project(), prior.project());
assert_eq!(result.warnings(), prior.warnings());
assert_eq!(result.phase_state().1, prior.phase_state().1);
}
assert_eq!(owner.bundle, original);
}
// Text-only and phase entry acceptance retain their old checkpoints.
for cap in [12, 11] {
let owner = held(bundle(""), cap);
let seed = parse_with_profile(
&owner.bundle,
owner.policy.project_limits,
owner.policy.limits,
)
.unwrap();
assert!(seed.budget.producer_entries().is_none());
let result = parse_entry_producers(CarriedInput::borrowed(&owner.bundle, &seed));
if cap == 12 {
assert_eq!(result.unwrap().metadata, 11);
} else {
diagnostic(&failed(result), "main.arcadia", 0, 1);
}
}
let low = held(
bundle("oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { }"),
4,
);
diagnostic(
&failed(parse_entry_producers(low.input())),
"main.arcadia",
75,
76,
);
diagnostic(
&failed(parse_text_producers(low.input())),
"main.arcadia",
0,
1,
);
diagnostic(&failed(parse_carried(low.input())), "main.arcadia", 0, 1);
}
@@ -0,0 +1,394 @@
//! Independent literal/storage oracles; expected text is not generated by parsing.
use super::*;
use crate::language::raw_tokens::{Class as RawClass, Scanner, StringEvent};
fn doc(body: &str, name: &str) -> String {
format!(
"arcadia 8;\nproject p \"{name}\" id \"00000000-0000-0000-0000-000000000001\" {{\n{body}\n}}\n"
)
}
fn bundle(body: &str, name: &str) -> SourceBundle {
SourceBundle {
entry: "main.arcadia".into(),
files: BTreeMap::from([("main.arcadia".into(), doc(body, name))]),
}
}
fn held(bundle: SourceBundle, entries: usize, bytes: usize) -> RetainedSnapshot {
RetainedSnapshot::fixture(
bundle,
SnapshotPolicy {
limits: Limits {
entries,
bytes,
..Limits::source8(ParseLimits::UNLIMITED)
},
project_limits: ProjectLimits::default(),
},
)
.unwrap()
}
fn failed(result: Result<ParsedMixedSyntax<'_>, Vec<SourceDiagnostic>>) -> SourceDiagnostic {
let errors = match result {
Ok(_) => panic!("unexpected syntax success"),
Err(e) => e,
};
assert_eq!(errors.len(), 1);
errors.into_iter().next().unwrap()
}
fn expected(name: &str, description: &str) -> crate::Project {
crate::Project {
schema_version: 8,
id: Id::parse_str("00000000-0000-0000-0000-000000000001").unwrap(),
alias: "p".into(),
name: name.into(),
description: description.into(),
definitions: BTreeMap::new(),
elements: BTreeMap::new(),
relations: BTreeMap::new(),
diagrams: BTreeMap::new(),
artifact_links: BTreeMap::new(),
traces: BTreeMap::new(),
}
}
#[test]
fn exact_real_owner_empty_and_escaped_peaks_reconcile_once() {
let escaped = "\\u0061".repeat(20);
for (b, entries, bytes, c, t, model) in [
(
bundle("", "Project"),
17,
146,
102,
8,
expected("Project", ""),
),
(
bundle(&format!("description \"{escaped}\";"), &escaped),
21,
491,
350,
41,
expected(&"a".repeat(20), &"a".repeat(20)),
),
] {
for cap in [bytes, bytes - 1] {
let owner = held(b.clone(), entries, cap);
assert_eq!(owner.retained.text, c);
let result = parse_text_producers(owner.input());
if cap == bytes {
let parsed = result.unwrap();
assert_eq!(parsed.budget.producer_text(), Some((t, 0)));
let usage = parsed.budget.usage;
let numeric = parsed.budget.numeric_units();
assert_eq!(
project::text_producer_inventory(&parsed).unwrap().text,
c + t
);
let resolved = project::resolve(parsed).unwrap();
assert_eq!(resolved.project(), &model);
assert_eq!(resolved.phase_state().1.text, c + t);
assert_eq!(resolved.phase_state().0.budget.usage, usage);
assert_eq!(resolved.phase_state().0.budget.numeric_units(), numeric);
let legacy =
project::resolve(parse(&owner.bundle, ProjectLimits::default()).unwrap())
.unwrap();
assert_eq!(legacy.project(), &model);
assert_eq!(legacy.phase_state().0.budget.usage, usage);
} else {
let e = failed(result);
assert_eq!(e.path, "main.arcadia");
assert!(e.diagnostic.message.contains("byte limit exceeded"));
let start = if c == 102 {
34
} else {
b.files["main.arcadia"].find("description \"").unwrap() + 12
};
assert_eq!(
e.diagnostic.span,
Span {
start,
end: start + 1
}
);
}
assert_eq!(owner.bundle, b);
}
}
let owner = held(bundle("", "Project"), 17, 112);
let syntax = parse_carried(owner.input()).unwrap();
assert!(syntax.budget.producer_text().is_none());
assert_eq!(project::resolve(syntax).unwrap().phase_state().1.text, 110);
}
fn multi() -> SourceBundle {
let mut b = bundle("include \"part.arcadia\";", &"a".repeat(20));
b.files.insert("part.arcadia".into(),"include \"\\u006c\\u0061\\u0073\\u0074\\u002e\\u0061\\u0072\\u0063\\u0061\\u0064\\u0069\\u0061\";\r\n".into());
b.files.insert("last.arcadia".into(), "//é\r\n".into());
b
}
#[test]
fn multifile_retained_text_overlaps_scanner_and_preserves_complete_result() {
for cap in [358, 357] {
let owner = held(multi(), 29, cap);
assert_eq!(
owner.retained,
Slots {
entries: 3,
occurrences: 0,
text: 253
}
);
let result = parse_text_producers(owner.input());
if cap == 358 {
let parsed = result.unwrap();
assert_eq!(parsed.budget.producer_text(), Some((45, 0)));
assert_eq!(
(
parsed.budget.usage.bytes,
parsed.budget.usage.tokens,
parsed.budget.usage.entries
),
(205, 31, 3)
);
let resolved = project::resolve(parsed).unwrap();
assert_eq!(
resolved.phase_state().1,
Slots {
entries: 25,
occurrences: 0,
text: 298
}
);
assert_eq!(resolved.project(), &expected(&"a".repeat(20), ""));
} else {
let e = failed(result);
assert_eq!(e.path, "part.arcadia");
assert_eq!(e.diagnostic.span, Span { start: 8, end: 9 });
assert!(e.diagnostic.message.contains("byte limit exceeded"));
}
}
// Arithmetic seam replay of independent C253/T33, not owner provenance.
// No grammar runs here, so this refusal specifically proves scanner timing.
let b = multi();
let mut budget = Budget::new(Limits {
bytes: 357,
..Limits::source8(ParseLimits::UNLIMITED)
})
.unwrap();
budget.enable_text_producers(253).unwrap();
budget.own_text(33, Span::default()).unwrap();
let e = lex::tokens(&b.files["part.arcadia"], &mut budget).unwrap_err();
assert_eq!(e.span, Span { start: 8, end: 82 });
assert_eq!(budget.producer_text(), Some((33, 0)));
}
#[test]
fn projection_and_three_uuid_consumers_reconcile_nested_copy_inventory() {
let activity = "oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { description \"dddddddddddddddddddddddddddddddddddddddd\"; }";
for cap in [304, 303] {
let owner = held(bundle(activity, "Project"), 1000, cap);
assert_eq!(owner.retained.text, 221);
let result = parse_text_producers(owner.input());
if cap == 304 {
let syntax = result.unwrap();
assert_eq!(syntax.budget.producer_text(), Some((83, 0)));
assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, 304);
} else {
assert_eq!(
failed(result).diagnostic.span,
Span {
start: 137,
end: 138
}
);
}
}
let body = "oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { payload { ref @p; old uuid \"00000000-0000-0000-0000-000000000001\"; }; }\ncalculation c \"C\" id \"00000000-0000-0000-0000-000000000003\" { input x id \"00000000-0000-0000-0000-000000000001\" : Number; result Number; body $x; }";
for cap in [453, 452] {
let owner = held(bundle(body, "Project"), 1000, cap);
assert_eq!(owner.retained.text, 383);
let result = parse_text_producers(owner.input());
if cap == 453 {
let syntax = result.unwrap();
assert_eq!(syntax.budget.producer_text(), Some((49, 0)));
assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, 432);
// Arbitrary payload is deliberately syntax-only, not model admission.
} else {
assert_eq!(failed(result).diagnostic.span.start, 282);
}
}
}
#[test]
fn raw_bounds_hook_order_refund_and_invalid_legacy_precedence() {
for (raw, decoded, payload) in [
(r#""""#, "", 0),
(r#""a""#, "a", 1),
(r#""é""#, "é", 2),
(r#""\\""#, "\\", 2),
(r#""\n""#, "\n", 2),
(r#""\u0061""#, "a", 6),
(r#""\u96ea""#, "雪", 6),
(r#""\uD83D\uDE00""#, "😀", 12),
] {
let mut events = Vec::new();
let token = Scanner::new(raw)
.next_with_strings(|event| {
events.push(event);
Ok::<(), ()>(())
})
.unwrap()
.unwrap();
assert_eq!(token.class, RawClass::String);
assert_eq!(
events,
vec![
StringEvent::Reserve {
span: Span {
start: 0,
end: raw.len()
},
payload
},
StringEvent::Released { payload }
]
);
// Arithmetic-only bounds; baseline100 is not actual-owner provenance.
let mut budget = Budget::new(Limits {
bytes: 100 + payload,
..Limits::source8(ParseLimits::UNLIMITED)
})
.unwrap();
budget.enable_text_producers(100).unwrap();
let value = budget.quoted_text(raw, token.span).unwrap();
assert_eq!(value, decoded);
assert_eq!(budget.producer_text(), Some((decoded.len(), 0)));
let bytes = value.len();
drop(value);
budget.release_text(bytes);
assert_eq!(budget.producer_text(), Some((0, 0)));
if payload > 0 {
let mut budget = Budget::new(Limits {
bytes: 100 + payload - 1,
..Limits::source8(ParseLimits::UNLIMITED)
})
.unwrap();
budget.enable_text_producers(100).unwrap();
assert!(budget.quoted_text(raw, token.span).is_err());
assert_eq!(budget.producer_text(), Some((0, 0)));
}
}
for raw in [r#""\q""#, r#""\uD800""#, r#""\uDE00""#] {
assert_eq!(Scanner::new(raw).next().unwrap().class, RawClass::Error);
let mut events = Vec::new();
let token = Scanner::new(raw)
.next_with_strings(|e| {
events.push(e);
Ok::<(), ()>(())
})
.unwrap()
.unwrap();
assert_eq!(token.class, RawClass::Error);
assert_eq!(events.len(), 2);
let mut calls = 0;
let result = Scanner::new(raw).next_with_strings(|_| {
calls += 1;
Err::<(), _>("reserve refused")
});
assert_eq!(result.unwrap_err(), "reserve refused");
assert_eq!(calls, 1);
}
let mut calls = 0;
assert_eq!(
Scanner::new("\"unclosed")
.next_with_strings(|_| {
calls += 1;
Ok::<(), ()>(())
})
.unwrap()
.unwrap()
.class,
RawClass::Error
);
assert_eq!(calls, 0);
let b = bundle(&format!("description \"{}\";", "\\q".repeat(60)), "Project");
let bytes = b.files["main.arcadia"].len() + 24 + 119;
let owner = held(b, 1000, bytes);
assert!(
failed(parse_text_producers(owner.input()))
.diagnostic
.message
.contains("byte limit exceeded")
);
assert_eq!(
failed(parse_carried(owner.input())).diagnostic.message,
"Invalid source token."
);
assert_eq!(
failed(parse(&owner.bundle, ProjectLimits::default()))
.diagnostic
.message,
"Invalid source token."
);
}
#[test]
fn explicit_zero_mode_complete_oa_sa_parity_and_checked_arithmetic() {
for (work, property) in [
("oa_activity", "oa_property"),
("sa_function", "sa_property"),
] {
let body = format!(
"calculation c \"C\" id \"00000000-0000-0000-0000-000000000003\" {{ input x id \"00000000-0000-0000-0000-000000000004\" : Number; result Number; body $x; }}\n{property} v \"V\" id \"00000000-0000-0000-0000-000000000002\" {{ subject @w; type Number; base @c(x: 1); }}\n{work} w \"W\" id \"00000000-0000-0000-0000-000000000005\" {{}}"
);
let b = bundle(&body, "Project");
let legacy = parse(&b, ProjectLimits::default()).unwrap();
let usage = legacy.budget.usage;
let numeric = legacy.budget.numeric_units();
let syntax = parse_text_producers(CarriedInput::borrowed(&b, &legacy)).unwrap();
let t = syntax.budget.producer_text().unwrap();
assert_eq!(t.1, 0);
assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, t.0);
let result = project::resolve(syntax).unwrap();
let prior = project::resolve(legacy).unwrap();
assert_eq!(result.project(), prior.project());
assert_eq!(result.warnings(), prior.warnings());
assert_eq!(result.phase_state().0.budget.usage, usage);
assert_eq!(result.phase_state().0.budget.numeric_units(), numeric);
assert_eq!(result.phase_state().1, prior.phase_state().1);
}
// Arithmetic faults, not fabricated large allocations or owner provenance.
let mut budget = Budget::new(Limits::source8(ParseLimits::UNLIMITED)).unwrap();
budget.enable_text_producers(1).unwrap();
assert!(budget.own_text(usize::MAX, Span::default()).is_err());
assert_eq!(budget.producer_text(), Some((0, 0)));
assert!(
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| budget.release_text(1))).is_err()
);
}
#[test]
fn duplicate_descriptions_refuse_before_a_second_grammar_decode() {
let text = "d".repeat(100);
for (body, extra, message) in [
(
format!("description \"{text}\"; description \"{text}\";"),
108,
"Duplicate project description.",
),
(
format!(
"calculation c \"C\" id \"00000000-0000-0000-0000-000000000002\" {{ description \"{text}\"; description \"{text}\"; }}"
),
110,
"Duplicate field 'description'.",
),
] {
let b = bundle(&body, "Project");
// Independently counted C is source plus the two12-byte owned paths;
// first description needs108/110, a second decoded copy would not fit.
let bytes = b.files["main.arcadia"].len() + 24 + extra;
let owner = held(b, 1000, bytes);
let e = failed(parse_text_producers(owner.input()));
assert_eq!(e.diagnostic.message, message);
assert_eq!(
failed(parse_carried(owner.input())).diagnostic.message,
message
);
}
}
@@ -11,6 +11,17 @@ use crate::{Id, Kind, Project, RelationKind, analysis, language, validation};
use std::collections::BTreeMap;
pub(in crate::language::analysis_source) mod accounting;
use accounting::{Ledger, Slots, add};
#[cfg(test)]
pub(super) fn text_producer_inventory(syntax: &ParsedMixedSyntax<'_>) -> Result<Slots, Failure> {
let mut ledger = Ledger {
live: syntax
.carried
.as_ref()
.map_or_else(Slots::default, |g| g.slots()),
};
accounting::syntax(syntax, &mut ledger)?;
Ok(ledger.live)
}
pub(in crate::language::analysis_source) mod ordinary_index;
/// Read-only authority: every component originates in the consumed syntax.
@@ -104,9 +104,14 @@ struct Walk<'a, 'b> {
budget: &'b Budget,
work: Vec<Item<'a>>,
lower: LowerStorage,
text_total: Option<usize>,
}
impl<'a> Walk<'a, '_> {
fn text(&mut self, text: &str, at: Span) -> Result<(), TypedError> {
if let Some(total) = &mut self.text_total {
*total = add(*total, text.len(), at)?;
return Ok(());
}
self.ledger
.reserve(self.budget, Slots::text(text.len()), at)
}
@@ -236,6 +241,19 @@ pub(super) fn syntax(
ledger: &mut Ledger,
) -> Result<LowerStorage, Failure> {
let at = Span::default();
let producer_text = syntax.budget.producer_text();
if let Some((text, temporary)) = producer_text {
if temporary != 0 {
return Err(Failure::typed(
syntax,
None,
error(at, "Unreleased syntax text temporary."),
));
}
ledger
.reserve(&syntax.budget, Slots::text(text), at)
.map_err(|e| Failure::typed(syntax, None, e))?;
}
ledger
.reserve(
&syntax.budget,
@@ -251,6 +269,7 @@ pub(super) fn syntax(
budget: &syntax.budget,
work: Vec::new(),
lower: LowerStorage::default(),
text_total: producer_text.map(|_| 0),
};
for (file, unit) in syntax.files.iter().enumerate() {
let mut count = || -> Result<(), TypedError> {
@@ -348,5 +367,12 @@ pub(super) fn syntax(
};
count().map_err(|e| Failure::typed(syntax, Some(file), e))?;
}
if walk.text_total != producer_text.map(|(text, _)| text) {
return Err(Failure::typed(
syntax,
None,
error(at, "Producer text differs from retained syntax inventory."),
));
}
Ok(walk.lower)
}
+38 -8
View File
@@ -32,14 +32,31 @@ impl<'a> Scanner<'a> {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum StringEvent {
Reserve { span: Span, payload: usize },
Released { payload: usize },
}
impl Iterator for Scanner<'_> {
type Item = Token;
fn next(&mut self) -> Option<Token> {
match self.next_with_strings(|_| Ok::<(), std::convert::Infallible>(())) {
Ok(token) => token,
Err(never) => match never {},
}
}
}
impl Scanner<'_> {
pub(super) fn next_with_strings<E>(
&mut self,
mut hook: impl FnMut(StringEvent) -> Result<(), E>,
) -> Result<Option<Token>, E> {
let source = self.source;
let bytes = source.as_bytes();
let start = self.cursor;
let first = *bytes.get(start)?;
let Some(&first) = bytes.get(start) else {
return Ok(None);
};
let mut legacy_number_part = false;
let class = if matches!(first, b' ' | b'\t' | b'\r' | b'\n') {
self.cursor += 1;
@@ -123,11 +140,24 @@ impl Iterator for Scanner<'_> {
break;
}
}
if closed && serde_json::from_str::<String>(&source[start..self.cursor]).is_ok() {
Class::String
let valid = if closed {
let payload = self.cursor - start - 2;
hook(StringEvent::Reserve {
span: Span {
start,
end: self.cursor,
},
payload,
})?;
// The temporary Result/String drops at this statement's end,
// before the refund hook runs.
let valid = serde_json::from_str::<String>(&source[start..self.cursor]).is_ok();
hook(StringEvent::Released { payload })?;
valid
} else {
Class::Error
}
false
};
if valid { Class::String } else { Class::Error }
} else if b"{}();,[]:=@$+-*/<>!".contains(&first) {
self.cursor += 1;
if matches!(first, b'<' | b'>' | b'=' | b'!') && bytes.get(self.cursor) == Some(&b'=') {
@@ -144,14 +174,14 @@ impl Iterator for Scanner<'_> {
legacy_number_part = first == b'.';
Class::Error
};
Some(Token {
Ok(Some(Token {
class,
span: Span {
start,
end: self.cursor,
},
legacy_number_part,
})
}))
}
}
@@ -0,0 +1,56 @@
# Private retained-entry producer prerequisite
Base `97a8703adc75467a7f8e75f5c6cb00ca542b707c`; implements the accepted
`harness/analysis-entry-producer-contract-20260925.md` in the parent source tree.
This is fixture-only phase/producer evidence, not emitted-output admission.
The explicit entry route also enables text producers and derives C and exact
policy from the existing carried guard. Budget adds only optional scalar C/M.
Existing legacy, carried-phase and text-only entry paths retain their checkpoints.
U keeps its original event sites and work-limit precedence; prospective aggregate
checks latch without changing totals. F is installed before traversal. Private
State helpers reserve unique value-end keys and lexical owner occurrences before
mutation. Successful grammar events count projection slots with checked addition.
The new route reserves token-end slots and projection together, without the old
borrowed-work counting Vec. A scalar quota checks each projected envelope, field
and actual value before growth. Synthetic bodies remain included in seven.
Per-file actual map/owner lengths and final independent metadata reconcile M;
resolver C then U+M transfer and text reconciliation are unchanged.
Six tests in `mixed::carried::entry_producer` cover independent literal E and H
syntax caps 13/12 and 25/24 separately from resolver 17/16 and 29/28, owner/value
prefix refusals 4/7, and the 344-byte nested syntax-only fixture (U13, M89, P30,
C1; 103/102). They check node/element envelopes, nested positions and owner order,
quota underflow/leftover, direct unchanged collections on refusal and repeated
map-key replacement. Labeled arithmetic controls exercise overflow, sticky-first
failure, prospective U and work-limit precedence. Whole OA/SA Project, warnings,
Usage, numeric work and retained totals are compared with text-only under actual
and explicit zero guards. Old-route diagnostics and original bytes are checked.
All byte policies are roomy 4096 so earlier text limits do not mask entry tests.
Source review identified a possible internal overflow call overwriting an already
latched error. The helper now returns the first failure; a direct fault control
checks that behavior. Repeated map keys now perform a zero-delta reservation
before replacement, preserving a latched refusal without modifying the map;
a direct control checks that branch too. These were corrected before compilation,
not a runtime bug
observation. Test review also caught that the independent inventory walker adds
its own existing scratch slots: exact-cap ordinary syntax controls remain at
29/37/103, while a separate roomy owner of the same literal runs that walk.
No live policy is widened and no exact-cap resolver success is implied.
No prior tests, stack sizes, policies, inherited failure dispositions
or unrelated formatting are changed. Only touched Rust files were directly
formatted with skip_children. Parent owns compiler, full disposition, Clippy,
format and prologue/stack acceptance; no runtime result is claimed here.
Allocator overhead, token storage under its existing token ceiling, delimiter and
other work stacks, numeric/decoder internals and model/validator heap remain
outside this retained metric. The extra shared projector wrapper and scalar
Budget/State fields require measured parent stack checks; no heap indirection or
larger stack is introduced. Final output acceptance remains a separate task.
Initial a629ae01 focused compilation failed before any test ran: the quota
control's unwrap_err required Debug on its successful EditDeclaration vector.
The test now matches Err explicitly and panics on unexpected Ok, preserving
the same assertion without changing production types. Original compiler output
and source-bound receipt remain with parent gate evidence.
@@ -0,0 +1,32 @@
# Private retained-syntax text producer prerequisite
2026-09-25. Candidate on source baseline `8d52dab97c92cc31baae9f20d8c324b55818aa5d`, selected root contract `harness/analysis-text-producer-contract-20260925.md` at local-only `17836587`. Root publication is parent-owned and server capacity is not reported recovered. This note records authored work, not compiler/runtime acceptance.
An explicit fixture-only `parse_text_producers` route derives bundle, guard and exact policies from CarriedInput. Existing legacy and carried-phase routes pass disabled mode. Budget adds only optional scalar carried/owned/temporary text counts; source/token/entry/reference/numeric Usage remains separate. Named storage checks use the inherited byte ceiling before copy/decode and preserve failed-reservation totals. Quoted text reserves raw payload P, reconciles decoded D<=P, and transfers the existing reservation continuously into owned text without a second fallible reservation. The conservative escaped-fit refusal is intentional only for this new route.
One shared raw scanner body has a bounded before-validation/refund hook. Its original Iterator API uses a no-op hook. A completed quoted token reserves before the existing serde validation; the temporary result String drops at the statement end before refund. Unclosed tokens skip decoding. New-mode storage refusal may precede malformed escape classification; old routes retain that classification. There is no new decoder or numeric conversion.
Typed and ordinary retained names, quoted fields, AST words/references/keys and projected key/reference copies reserve before creation. All three consumed UUID spelling sites drop then refund: ordinary optional IDs, typed identities, and ordinary HistoricalUuid. Nested field originals provisionally count as owned text alongside retained clones, then explicitly drop/refund; top-level fields and field_spans retain both copies. The declaration-kind spelling is borrowed from the original source. Existing duplicate description checks precede replacement/second grammar decoding.
Successful syntax has no outstanding decoder/scanner temporary reservation. Resolver seeds C once, transfers producer T once, and changes the existing complete text walk into an independent checked sum/equality check for this mode. Existing routes keep progressive text reservation. A cfg(test), read-only project inventory helper invokes that same bounded walk on syntax-only fixtures; it cannot replace bundle/policy/baseline authority.
Scope remains logical retained String lengths and named result temporaries. Decoder internal scratch/capacity, existing duplicate-name coordination copies, traversal/entry/metadata producer timing, model conversion/validation, Exact internals, allocator overhead and native stack retain their explicit separate domains. No PendingRename extraction, emitted-output parser transition, accepted result, public source8, persistence or history operation is added.
## Six focused authored tests
All new names are under `mixed::carried::text_producer`, selected by filter `text_producer`. Independent literal/count expectations came from the source-only text-producer oracle, not application outputs.
- Empty real owner E complete cap146/145, finalT8; escaped project cap491/490, finalT41; independent syntax inventory, complete expected Project and Usage parity, and unchanged carried-phase112 control.
- Three-file escaped include cap358/357, C253/T45, final resolved298 and entry25. A separately labeled scanner-only state replay distinguishes scanner reservation from the identical later grammar threshold.
- Projection-dominant syntax cap304/303, finalT83 and independent inventory304; nested/reference/all-three-UUID syntax fixture cap453/452, finalT49/inventory432. The arbitrary payload fixture is explicitly not asserted to admit a Project.
- Raw/escaped Unicode, simple escapes, surrogate pairs and empty strings: rawP cap/refund and scanner hook order; malformed escapes/surrogates, unclosed tokens and explicit legacy error precedence. Arithmetic baselines are labeled separately from real-owner provenance.
- Explicit zero-C stronger route OA/SA complete model, warnings, final retained storage, Usage and numeric-work parity; checked overflow/refund fault injections.
- Project and typed duplicate descriptions with enough room for the first grammar decode but not a second, asserting the existing duplicate errors rather than a false successful overwrite.
No old test or worker-stack allowance changed. Implementer ran no Cargo, test executable or rustfmt; parent owns formatting, compiler, prologue/headroom and runtime gates. Independent production review found no initial semantic blocker; final review binds parent-formatted source and authored tests before commit. Preserve all prior recorded failures and compare actual full-suite dispositions rather than claiming they are unchanged from source inspection.
Parent gates at initial source6b0ef99 passed all six focused tests and completed
the full library with259 passed /16 inherited failures. Strict Clippy rejected
the ordinary quoted hook's is_some/expect pattern. The successor uses direct
if-let matching with the same token span and decoder; no tests, limits or
accounting operations changed. Final source gates remain separately required.