diff --git a/crates/oa-core/src/language.rs b/crates/oa-core/src/language.rs index 1b688fff..0321cfc6 100644 --- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -1018,6 +1018,16 @@ fn edit_positions_from_tokens( token: impl Fn(usize) -> Lexeme, count: usize, value_ends: Option<&BTreeMap>, +) -> Result, Box> { + edit_positions_from_tokens_with_text(source, unit, token, count, value_ends, |_, _| Ok(())) +} +fn edit_positions_from_tokens_with_text( + source: &str, + unit: &SourceUnit, + token: impl Fn(usize) -> Lexeme, + count: usize, + value_ends: Option<&BTreeMap>, + text: impl FnMut(usize, Span) -> Result<(), Box>, ) -> Result, Box> { let mut ends: Vec<_> = (0..count).map(|at| token(at).span.end).collect(); let mut stack = Vec::new(); @@ -1029,12 +1039,15 @@ fn edit_positions_from_tokens( _ => {} } } - struct Positions<'a, F> { + struct Positions<'a, F, H> { token_at: F, ends: Vec, value_ends: Option<&'a BTreeMap>, + text: H, } - impl Lexeme> Positions<'_, F> { + impl Lexeme, H: FnMut(usize, Span) -> Result<(), Box>> + Positions<'_, F, H> + { fn token(&self, start: usize) -> usize { let mut low = 0; let mut high = self.ends.len(); @@ -1048,24 +1061,28 @@ fn edit_positions_from_tokens( } low } - fn fields(&self, fields: &BTreeMap) -> BTreeMap { + fn fields( + &mut self, + fields: &BTreeMap, + ) -> Result, Box> { fields .iter() .map(|(key, ast)| { - let value = self.value(ast); + let value = self.value(ast)?; let start = (self.token_at)(self.token(value.span.start) - 1).span.start; let end = (self.token_at)(self.token(value.span.end)).span.end; - ( + (self.text)(key.len(), Span { start, end })?; + Ok(( key.clone(), EditField { span: Span { start, end }, value, }, - ) + )) }) .collect() } - fn value(&self, ast: &Ast) -> EditValue { + fn value(&mut self, ast: &Ast) -> Result> { let index = self.token(ast.offset()); let span = Span { start: ast.offset(), @@ -1080,21 +1097,28 @@ fn edit_positions_from_tokens( }), }; let kind = match ast { - Ast::Object(fields, _) => EditKind::Record(self.fields(fields)), - Ast::Array(items, _) => { - EditKind::List(items.iter().map(|item| self.value(item)).collect()) + Ast::Object(fields, _) => EditKind::Record(self.fields(fields)?), + Ast::Array(items, _) => EditKind::List( + items + .iter() + .map(|item| self.value(item)) + .collect::>()?, + ), + Ast::Reference(alias, _) => { + (self.text)(alias.len(), span)?; + EditKind::Reference(alias.clone()) } - Ast::Reference(alias, _) => EditKind::Reference(alias.clone()), Ast::HistoricalUuid(id, _) => EditKind::HistoricalUuid(*id), _ => EditKind::Atom, }; - EditValue { span, kind } + Ok(EditValue { span, kind }) } } - let positions = Positions { + let mut positions = Positions { token_at: &token, ends, value_ends, + text, }; unit.declarations .iter() @@ -1132,7 +1156,7 @@ fn edit_positions_from_tokens( start: declaration.body.start - 1, end: declaration.body.end + 1, }, - kind: EditKind::Record(positions.fields(&declaration.fields)), + kind: EditKind::Record(positions.fields(&declaration.fields)?), }, }, )) @@ -1196,11 +1220,24 @@ impl Parser<'_> { if self.current().class != TokenClass::String { return Err(self.error("Expected a double-quoted string.")); } + #[cfg(test)] + if self.mixed.is_some() { + let span = self.current().span; + let raw = &self.source[span.start..span.end]; + let value = self + .mixed + .as_mut() + .expect("mixed quoted consumer") + .quoted_text(raw, span)?; + self.cursor += 1; + return Ok(value); + } let value = serde_json::from_str(self.text()).unwrap(); self.cursor += 1; Ok(value) } fn identifier(&mut self) -> Result { + self.own_text(self.text().len(), self.current().span)?; let value = self.text().to_owned(); if !crate::valid_alias(&value) { return Err(self.error(std::format!("Invalid or reserved alias '{value}'."))); @@ -1230,11 +1267,32 @@ impl Parser<'_> { message: "Expected UUID in 8-4-4-4-12 hexadecimal form.".into(), }); } - Id::parse_str(&value).map_err(|_| Error { + let id = Id::parse_str(&value).map_err(|_| Error { code: "invalid_source", offset, message: "Invalid UUID identity.".into(), - }) + })?; + let bytes = value.len(); + drop(value); + self.release_text(bytes); + Ok(id) + } + fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> { + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.own_text(bytes, span)?; + } + #[cfg(not(test))] + let _ = (bytes, span); + Ok(()) + } + fn release_text(&mut self, bytes: usize) { + #[cfg(test)] + if let Some(mixed) = &mut self.mixed { + mixed.release_text(bytes); + } + #[cfg(not(test))] + let _ = bytes; } fn fields(&mut self) -> Result, Error> { self.take("{")?; @@ -1242,6 +1300,13 @@ impl Parser<'_> { while self.text() != "}" { let offset = self.current().span.start; let (key, value, _) = self.field()?; + self.own_text( + key.len(), + Span { + start: offset, + end: offset, + }, + )?; if fields.insert(key.clone(), value).is_some() { return Err(Error { code: "invalid_source", @@ -1249,12 +1314,16 @@ impl Parser<'_> { message: std::format!("Duplicate field '{key}'."), }); } + let bytes = key.len(); + drop(key); + self.release_text(bytes); } self.take("}")?; Ok(fields) } fn field(&mut self) -> Result<(String, Ast, Span), Error> { self.syntax_entry()?; + self.own_text(self.text().len(), self.current().span)?; let key = self.text().to_owned(); if !matches!( self.current().class, @@ -1328,6 +1397,9 @@ impl Parser<'_> { offset, message: "Expected a UUID after uuid.".into(), })?; + let bytes = value.len(); + drop(value); + self.release_text(bytes); Ok(Ast::HistoricalUuid(id, offset)) } "@" => { @@ -1368,6 +1440,7 @@ impl Parser<'_> { Ok(Ast::Literal(value, offset)) } TokenClass::Identifier | TokenClass::Keyword => { + self.own_text(self.text().len(), self.current().span)?; let value = self.text().to_owned(); self.cursor += 1; Ok(Ast::Word(value, offset)) @@ -1514,6 +1587,7 @@ impl Parser<'_> { } } else { let (key, value, span) = self.field()?; + self.own_text(key.len(), span)?; let declaration = &mut declarations[owner]; if declaration.fields.insert(key.clone(), value).is_some() { return Err(Error { @@ -1584,8 +1658,8 @@ impl Parser<'_> { return Err(self.error("Use the short source8 analysis declaration word.")); } self.syntax_entry()?; - let word = self.text().to_owned(); - let kind = declaration_kind(&word) + let word = self.current().text(self.source); + let kind = declaration_kind(word) .ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?; #[cfg(test)] if let Some(mixed) = &mut self.mixed { diff --git a/crates/oa-core/src/language/analysis_source/budget.rs b/crates/oa-core/src/language/analysis_source/budget.rs index c0cf1124..6ede7fa9 100644 --- a/crates/oa-core/src/language/analysis_source/budget.rs +++ b/crates/oa-core/src/language/analysis_source/budget.rs @@ -38,6 +38,12 @@ pub struct Budget { pub usage: Usage, numeric: WorkMeter, failed: Option, + text: Option, +} +struct ProducerText { + carried: usize, + owned: usize, + temporary: usize, } impl Budget { pub fn new(limits: Limits) -> Result { @@ -62,8 +68,97 @@ impl Budget { .map_err(|e| error(Span::default(), e.to_string()))?, ), failed: None, + text: None, }) } + pub(super) fn enable_text_producers(&mut self, carried: usize) -> Result<(), Error> { + self.index_bytes(carried, Span::default())?; + assert!(self.text.is_none()); + self.text = Some(ProducerText { + carried, + owned: 0, + temporary: 0, + }); + Ok(()) + } + pub(super) fn producer_text(&self) -> Option<(usize, usize)> { + self.text.as_ref().map(|t| (t.owned, t.temporary)) + } + fn reserve_text(&mut self, bytes: usize, temporary: bool, span: Span) -> Result<(), Error> { + let Some(t) = &self.text else { + return Ok(()); + }; + let total = t + .carried + .checked_add(t.owned) + .and_then(|n| n.checked_add(t.temporary)) + .and_then(|n| n.checked_add(bytes)) + .ok_or_else(|| error(span, "Source8 producer text size overflow."))?; + self.index_bytes(total, span)?; + let t = self.text.as_mut().expect("enabled producer text"); + if temporary { + t.temporary += bytes; + } else { + t.owned += bytes; + } + Ok(()) + } + pub(super) fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> { + self.reserve_text(bytes, false, span) + } + pub(super) fn temporary_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> { + self.reserve_text(bytes, true, span) + } + pub(super) fn release_text(&mut self, bytes: usize) { + if let Some(t) = &mut self.text { + t.owned = t.owned.checked_sub(bytes).expect("owned text reservation"); + } + } + pub(super) fn release_temporary_text(&mut self, bytes: usize) { + if let Some(t) = &mut self.text { + t.temporary = t + .temporary + .checked_sub(bytes) + .expect("temporary text reservation"); + } + } + pub(super) fn quoted_text(&mut self, raw: &str, span: Span) -> Result { + if self.text.is_none() { + return serde_json::from_str(raw).map_err(|_| error(span, "Invalid quoted string.")); + } + let payload = raw + .len() + .checked_sub(2) + .ok_or_else(|| error(span, "Invalid quoted text extent."))?; + self.temporary_text(payload, span)?; + let result = serde_json::from_str::(raw); + let value = match result { + Ok(value) => value, + Err(_) => { + self.release_temporary_text(payload); + return Err(error(span, "Invalid quoted string.")); + } + }; + if value.len() > payload { + drop(value); + self.release_temporary_text(payload); + return Err(error(span, "Decoded text exceeds its payload reservation.")); + } + // Keep the reservation continuous while moving decoded bytes into T. + // D <= P and the original checked aggregate bound prove both sums fit. + let t = self.text.as_mut().expect("enabled producer text"); + let temporary = t + .temporary + .checked_sub(payload) + .expect("quoted text reservation"); + let owned = t + .owned + .checked_add(value.len()) + .expect("bounded decoded transfer"); + t.temporary = temporary; + t.owned = owned; + Ok(value) + } /// Immutable exact policy snapshot for private deliberate-snapshot continuation. pub(super) fn limits(&self) -> Limits { self.limits diff --git a/crates/oa-core/src/language/analysis_source/cursor.rs b/crates/oa-core/src/language/analysis_source/cursor.rs index 84723258..82e5d4d4 100644 --- a/crates/oa-core/src/language/analysis_source/cursor.rs +++ b/crates/oa-core/src/language/analysis_source/cursor.rs @@ -74,6 +74,7 @@ impl Cursor<'_, '_> { { return Err(error(span, "Expected a valid reference or parameter name.")); } + self.budget.own_text(self.text().len(), span)?; let name = self.text().to_owned(); self.cursor += 1; Ok(Reference { name, span }) @@ -87,8 +88,8 @@ impl Cursor<'_, '_> { { return Err(error(span, "Expected a quoted string.")); } - let value = - serde_json::from_str(self.text()).map_err(|_| error(span, "Invalid quoted string."))?; + let raw = &self.source[span.start..span.end]; + let value = self.budget.quoted_text(raw, span)?; self.cursor += 1; Ok(value) } diff --git a/crates/oa-core/src/language/analysis_source/declarations.rs b/crates/oa-core/src/language/analysis_source/declarations.rs index 64af3939..f1898394 100644 --- a/crates/oa-core/src/language/analysis_source/declarations.rs +++ b/crates/oa-core/src/language/analysis_source/declarations.rs @@ -426,6 +426,9 @@ impl Cursor<'_, '_> { )); } let id = Id::parse_str(&spelling).map_err(|_| error(literal, "Invalid UUID identity."))?; + let bytes = spelling.len(); + drop(spelling); + self.budget.release_text(bytes); Ok(IdentitySyntax { value: Some(id), keyword: Some(keyword), diff --git a/crates/oa-core/src/language/analysis_source/lex.rs b/crates/oa-core/src/language/analysis_source/lex.rs index 335d47c5..78fe4a3d 100644 --- a/crates/oa-core/src/language/analysis_source/lex.rs +++ b/crates/oa-core/src/language/analysis_source/lex.rs @@ -1,5 +1,5 @@ //! Source8 significant-token adapter over the shared raw scanner. -use super::super::raw_tokens::{Class as RawClass, Scanner}; +use super::super::raw_tokens::{Class as RawClass, Scanner, StringEvent}; use super::{Budget, Error, Span, error}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -33,7 +33,20 @@ pub fn tokens(source: &str, budget: &mut Budget) -> Result, Error> { start: offset, end: offset, })?; - let token = scanner.next().expect("source remains"); + let token = if budget.producer_text().is_some() { + scanner.next_with_strings(|event| { + match event { + StringEvent::Reserve { span, payload } => { + budget.temporary_text(payload, span)? + } + StringEvent::Released { payload } => budget.release_temporary_text(payload), + } + Ok::<(), Error>(()) + })? + } else { + scanner.next() + } + .expect("source remains"); offset = token.span.end; let class = match token.class { RawClass::Word => Class::Word, diff --git a/crates/oa-core/src/language/analysis_source/mixed.rs b/crates/oa-core/src/language/analysis_source/mixed.rs index 8c822750..cc0d130a 100644 --- a/crates/oa-core/src/language/analysis_source/mixed.rs +++ b/crates/oa-core/src/language/analysis_source/mixed.rs @@ -93,6 +93,19 @@ fn lexeme(token: lex::Token) -> Lexeme { } impl State<'_> { + pub(in crate::language) fn own_text(&mut self, bytes: usize, span: Span) -> Result<(), Error> { + self.budget.own_text(bytes, span).map_err(ordinary_error) + } + pub(in crate::language) fn release_text(&mut self, bytes: usize) { + self.budget.release_text(bytes); + } + pub(in crate::language) fn quoted_text( + &mut self, + raw: &str, + span: Span, + ) -> Result { + self.budget.quoted_text(raw, span).map_err(ordinary_error) + } pub(in crate::language) fn token(&self, at: usize) -> Option { self.tokens.get(at).copied().map(lexeme) } @@ -250,7 +263,7 @@ fn parse_with_profile( limits: ProjectLimits, profile: Limits, ) -> Result, Vec> { - parse_impl(bundle, limits, profile, None) + parse_impl(bundle, limits, profile, None, false) } fn parse_impl<'s>( @@ -258,6 +271,7 @@ fn parse_impl<'s>( limits: ProjectLimits, profile: Limits, carried: Option>, + text_producers: bool, ) -> Result, Vec> { let entry_source = bundle.files.get(&bundle.entry).map_or("", String::as_str); let at_entry = |error| located(&bundle.entry, entry_source, ordinary_error(error)); @@ -279,6 +293,14 @@ fn parse_impl<'s>( ))); } let mut budget = Budget::new(profile).map_err(at_entry)?; + if text_producers { + let input = carried + .as_ref() + .expect("text mode requires carried authority"); + budget + .enable_text_producers(input.slots().text) + .map_err(at_entry)?; + } if let Some(input) = &carried { input.check(&budget, 0, 0).map_err(at_entry)?; } @@ -355,12 +377,18 @@ fn parse_impl<'s>( .lowering_entries(metadata, Span::default()) .map_err(failure)?; } - let positions = language::edit_positions_from_tokens( + let positions = language::edit_positions_from_tokens_with_text( source, &ordinary, |at| lexeme(tokens[at]), tokens.len(), Some(&state.value_ends), + |bytes, span| { + state + .budget + .own_text(bytes, span) + .map_err(|e| Box::new(language::diagnostic(source, ordinary_error(e)))) + }, ) .map_err(|e| { vec![SourceDiagnostic { @@ -393,6 +421,9 @@ fn parse_impl<'s>( } traversal.finish()?; budget.active().map_err(at_entry)?; + if let Some((_, temporary)) = budget.producer_text() { + assert_eq!(temporary, 0, "completed syntax has no temporary text"); + } if let Some(input) = &carried { input .check(&budget, budget.usage.entries, metadata) diff --git a/crates/oa-core/src/language/analysis_source/mixed/carried.rs b/crates/oa-core/src/language/analysis_source/mixed/carried.rs index 357c077f..851aebaf 100644 --- a/crates/oa-core/src/language/analysis_source/mixed/carried.rs +++ b/crates/oa-core/src/language/analysis_source/mixed/carried.rs @@ -85,7 +85,28 @@ impl CarriedGuard<'_> { /// Input already binds the actual bundle; callers cannot supply a replacement. fn parse_carried(input: CarriedInput<'_>) -> Result, Vec> { let (bundle, policy, guard) = input.into_parts(); - parse_impl(bundle, policy.project_limits, policy.limits, Some(guard)) + parse_impl( + bundle, + policy.project_limits, + policy.limits, + Some(guard), + false, + ) +} + +/// Explicit stronger fixture route; the actual owner supplies all authority. +#[cfg(test)] +fn parse_text_producers( + input: CarriedInput<'_>, +) -> Result, Vec> { + let (bundle, policy, guard) = input.into_parts(); + parse_impl( + bundle, + policy.project_limits, + policy.limits, + Some(guard), + true, + ) } #[cfg(test)] @@ -139,3 +160,6 @@ impl<'s> CarriedInput<'s> { #[cfg(test)] mod tests; + +#[cfg(test)] +mod text_producer; diff --git a/crates/oa-core/src/language/analysis_source/mixed/carried/text_producer.rs b/crates/oa-core/src/language/analysis_source/mixed/carried/text_producer.rs new file mode 100644 index 00000000..1df254e9 --- /dev/null +++ b/crates/oa-core/src/language/analysis_source/mixed/carried/text_producer.rs @@ -0,0 +1,394 @@ +//! Independent literal/storage oracles; expected text is not generated by parsing. +use super::*; +use crate::language::raw_tokens::{Class as RawClass, Scanner, StringEvent}; +fn doc(body: &str, name: &str) -> String { + format!( + "arcadia 8;\nproject p \"{name}\" id \"00000000-0000-0000-0000-000000000001\" {{\n{body}\n}}\n" + ) +} +fn bundle(body: &str, name: &str) -> SourceBundle { + SourceBundle { + entry: "main.arcadia".into(), + files: BTreeMap::from([("main.arcadia".into(), doc(body, name))]), + } +} +fn held(bundle: SourceBundle, entries: usize, bytes: usize) -> RetainedSnapshot { + RetainedSnapshot::fixture( + bundle, + SnapshotPolicy { + limits: Limits { + entries, + bytes, + ..Limits::source8(ParseLimits::UNLIMITED) + }, + project_limits: ProjectLimits::default(), + }, + ) + .unwrap() +} +fn failed(result: Result, Vec>) -> SourceDiagnostic { + let errors = match result { + Ok(_) => panic!("unexpected syntax success"), + Err(e) => e, + }; + assert_eq!(errors.len(), 1); + errors.into_iter().next().unwrap() +} +fn expected(name: &str, description: &str) -> crate::Project { + crate::Project { + schema_version: 8, + id: Id::parse_str("00000000-0000-0000-0000-000000000001").unwrap(), + alias: "p".into(), + name: name.into(), + description: description.into(), + definitions: BTreeMap::new(), + elements: BTreeMap::new(), + relations: BTreeMap::new(), + diagrams: BTreeMap::new(), + artifact_links: BTreeMap::new(), + traces: BTreeMap::new(), + } +} +#[test] +fn exact_real_owner_empty_and_escaped_peaks_reconcile_once() { + let escaped = "\\u0061".repeat(20); + for (b, entries, bytes, c, t, model) in [ + ( + bundle("", "Project"), + 17, + 146, + 102, + 8, + expected("Project", ""), + ), + ( + bundle(&format!("description \"{escaped}\";"), &escaped), + 21, + 491, + 350, + 41, + expected(&"a".repeat(20), &"a".repeat(20)), + ), + ] { + for cap in [bytes, bytes - 1] { + let owner = held(b.clone(), entries, cap); + assert_eq!(owner.retained.text, c); + let result = parse_text_producers(owner.input()); + if cap == bytes { + let parsed = result.unwrap(); + assert_eq!(parsed.budget.producer_text(), Some((t, 0))); + let usage = parsed.budget.usage; + let numeric = parsed.budget.numeric_units(); + assert_eq!( + project::text_producer_inventory(&parsed).unwrap().text, + c + t + ); + let resolved = project::resolve(parsed).unwrap(); + assert_eq!(resolved.project(), &model); + assert_eq!(resolved.phase_state().1.text, c + t); + assert_eq!(resolved.phase_state().0.budget.usage, usage); + assert_eq!(resolved.phase_state().0.budget.numeric_units(), numeric); + let legacy = + project::resolve(parse(&owner.bundle, ProjectLimits::default()).unwrap()) + .unwrap(); + assert_eq!(legacy.project(), &model); + assert_eq!(legacy.phase_state().0.budget.usage, usage); + } else { + let e = failed(result); + assert_eq!(e.path, "main.arcadia"); + assert!(e.diagnostic.message.contains("byte limit exceeded")); + let start = if c == 102 { + 34 + } else { + b.files["main.arcadia"].find("description \"").unwrap() + 12 + }; + assert_eq!( + e.diagnostic.span, + Span { + start, + end: start + 1 + } + ); + } + assert_eq!(owner.bundle, b); + } + } + let owner = held(bundle("", "Project"), 17, 112); + let syntax = parse_carried(owner.input()).unwrap(); + assert!(syntax.budget.producer_text().is_none()); + assert_eq!(project::resolve(syntax).unwrap().phase_state().1.text, 110); +} +fn multi() -> SourceBundle { + let mut b = bundle("include \"part.arcadia\";", &"a".repeat(20)); + b.files.insert("part.arcadia".into(),"include \"\\u006c\\u0061\\u0073\\u0074\\u002e\\u0061\\u0072\\u0063\\u0061\\u0064\\u0069\\u0061\";\r\n".into()); + b.files.insert("last.arcadia".into(), "//é\r\n".into()); + b +} +#[test] +fn multifile_retained_text_overlaps_scanner_and_preserves_complete_result() { + for cap in [358, 357] { + let owner = held(multi(), 29, cap); + assert_eq!( + owner.retained, + Slots { + entries: 3, + occurrences: 0, + text: 253 + } + ); + let result = parse_text_producers(owner.input()); + if cap == 358 { + let parsed = result.unwrap(); + assert_eq!(parsed.budget.producer_text(), Some((45, 0))); + assert_eq!( + ( + parsed.budget.usage.bytes, + parsed.budget.usage.tokens, + parsed.budget.usage.entries + ), + (205, 31, 3) + ); + let resolved = project::resolve(parsed).unwrap(); + assert_eq!( + resolved.phase_state().1, + Slots { + entries: 25, + occurrences: 0, + text: 298 + } + ); + assert_eq!(resolved.project(), &expected(&"a".repeat(20), "")); + } else { + let e = failed(result); + assert_eq!(e.path, "part.arcadia"); + assert_eq!(e.diagnostic.span, Span { start: 8, end: 9 }); + assert!(e.diagnostic.message.contains("byte limit exceeded")); + } + } + // Arithmetic seam replay of independent C253/T33, not owner provenance. + // No grammar runs here, so this refusal specifically proves scanner timing. + let b = multi(); + let mut budget = Budget::new(Limits { + bytes: 357, + ..Limits::source8(ParseLimits::UNLIMITED) + }) + .unwrap(); + budget.enable_text_producers(253).unwrap(); + budget.own_text(33, Span::default()).unwrap(); + let e = lex::tokens(&b.files["part.arcadia"], &mut budget).unwrap_err(); + assert_eq!(e.span, Span { start: 8, end: 82 }); + assert_eq!(budget.producer_text(), Some((33, 0))); +} +#[test] +fn projection_and_three_uuid_consumers_reconcile_nested_copy_inventory() { + let activity = "oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { description \"dddddddddddddddddddddddddddddddddddddddd\"; }"; + for cap in [304, 303] { + let owner = held(bundle(activity, "Project"), 1000, cap); + assert_eq!(owner.retained.text, 221); + let result = parse_text_producers(owner.input()); + if cap == 304 { + let syntax = result.unwrap(); + assert_eq!(syntax.budget.producer_text(), Some((83, 0))); + assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, 304); + } else { + assert_eq!( + failed(result).diagnostic.span, + Span { + start: 137, + end: 138 + } + ); + } + } + let body = "oa_activity a \"A\" id \"00000000-0000-0000-0000-000000000002\" { payload { ref @p; old uuid \"00000000-0000-0000-0000-000000000001\"; }; }\ncalculation c \"C\" id \"00000000-0000-0000-0000-000000000003\" { input x id \"00000000-0000-0000-0000-000000000001\" : Number; result Number; body $x; }"; + for cap in [453, 452] { + let owner = held(bundle(body, "Project"), 1000, cap); + assert_eq!(owner.retained.text, 383); + let result = parse_text_producers(owner.input()); + if cap == 453 { + let syntax = result.unwrap(); + assert_eq!(syntax.budget.producer_text(), Some((49, 0))); + assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, 432); + // Arbitrary payload is deliberately syntax-only, not model admission. + } else { + assert_eq!(failed(result).diagnostic.span.start, 282); + } + } +} + +#[test] +fn raw_bounds_hook_order_refund_and_invalid_legacy_precedence() { + for (raw, decoded, payload) in [ + (r#""""#, "", 0), + (r#""a""#, "a", 1), + (r#""é""#, "é", 2), + (r#""\\""#, "\\", 2), + (r#""\n""#, "\n", 2), + (r#""\u0061""#, "a", 6), + (r#""\u96ea""#, "雪", 6), + (r#""\uD83D\uDE00""#, "😀", 12), + ] { + let mut events = Vec::new(); + let token = Scanner::new(raw) + .next_with_strings(|event| { + events.push(event); + Ok::<(), ()>(()) + }) + .unwrap() + .unwrap(); + assert_eq!(token.class, RawClass::String); + assert_eq!( + events, + vec![ + StringEvent::Reserve { + span: Span { + start: 0, + end: raw.len() + }, + payload + }, + StringEvent::Released { payload } + ] + ); + // Arithmetic-only bounds; baseline100 is not actual-owner provenance. + let mut budget = Budget::new(Limits { + bytes: 100 + payload, + ..Limits::source8(ParseLimits::UNLIMITED) + }) + .unwrap(); + budget.enable_text_producers(100).unwrap(); + let value = budget.quoted_text(raw, token.span).unwrap(); + assert_eq!(value, decoded); + assert_eq!(budget.producer_text(), Some((decoded.len(), 0))); + let bytes = value.len(); + drop(value); + budget.release_text(bytes); + assert_eq!(budget.producer_text(), Some((0, 0))); + if payload > 0 { + let mut budget = Budget::new(Limits { + bytes: 100 + payload - 1, + ..Limits::source8(ParseLimits::UNLIMITED) + }) + .unwrap(); + budget.enable_text_producers(100).unwrap(); + assert!(budget.quoted_text(raw, token.span).is_err()); + assert_eq!(budget.producer_text(), Some((0, 0))); + } + } + for raw in [r#""\q""#, r#""\uD800""#, r#""\uDE00""#] { + assert_eq!(Scanner::new(raw).next().unwrap().class, RawClass::Error); + let mut events = Vec::new(); + let token = Scanner::new(raw) + .next_with_strings(|e| { + events.push(e); + Ok::<(), ()>(()) + }) + .unwrap() + .unwrap(); + assert_eq!(token.class, RawClass::Error); + assert_eq!(events.len(), 2); + let mut calls = 0; + let result = Scanner::new(raw).next_with_strings(|_| { + calls += 1; + Err::<(), _>("reserve refused") + }); + assert_eq!(result.unwrap_err(), "reserve refused"); + assert_eq!(calls, 1); + } + let mut calls = 0; + assert_eq!( + Scanner::new("\"unclosed") + .next_with_strings(|_| { + calls += 1; + Ok::<(), ()>(()) + }) + .unwrap() + .unwrap() + .class, + RawClass::Error + ); + assert_eq!(calls, 0); + let b = bundle(&format!("description \"{}\";", "\\q".repeat(60)), "Project"); + let bytes = b.files["main.arcadia"].len() + 24 + 119; + let owner = held(b, 1000, bytes); + assert!( + failed(parse_text_producers(owner.input())) + .diagnostic + .message + .contains("byte limit exceeded") + ); + assert_eq!( + failed(parse_carried(owner.input())).diagnostic.message, + "Invalid source token." + ); + assert_eq!( + failed(parse(&owner.bundle, ProjectLimits::default())) + .diagnostic + .message, + "Invalid source token." + ); +} +#[test] +fn explicit_zero_mode_complete_oa_sa_parity_and_checked_arithmetic() { + for (work, property) in [ + ("oa_activity", "oa_property"), + ("sa_function", "sa_property"), + ] { + let body = format!( + "calculation c \"C\" id \"00000000-0000-0000-0000-000000000003\" {{ input x id \"00000000-0000-0000-0000-000000000004\" : Number; result Number; body $x; }}\n{property} v \"V\" id \"00000000-0000-0000-0000-000000000002\" {{ subject @w; type Number; base @c(x: 1); }}\n{work} w \"W\" id \"00000000-0000-0000-0000-000000000005\" {{}}" + ); + let b = bundle(&body, "Project"); + let legacy = parse(&b, ProjectLimits::default()).unwrap(); + let usage = legacy.budget.usage; + let numeric = legacy.budget.numeric_units(); + let syntax = parse_text_producers(CarriedInput::borrowed(&b, &legacy)).unwrap(); + let t = syntax.budget.producer_text().unwrap(); + assert_eq!(t.1, 0); + assert_eq!(project::text_producer_inventory(&syntax).unwrap().text, t.0); + let result = project::resolve(syntax).unwrap(); + let prior = project::resolve(legacy).unwrap(); + assert_eq!(result.project(), prior.project()); + assert_eq!(result.warnings(), prior.warnings()); + assert_eq!(result.phase_state().0.budget.usage, usage); + assert_eq!(result.phase_state().0.budget.numeric_units(), numeric); + assert_eq!(result.phase_state().1, prior.phase_state().1); + } + // Arithmetic faults, not fabricated large allocations or owner provenance. + let mut budget = Budget::new(Limits::source8(ParseLimits::UNLIMITED)).unwrap(); + budget.enable_text_producers(1).unwrap(); + assert!(budget.own_text(usize::MAX, Span::default()).is_err()); + assert_eq!(budget.producer_text(), Some((0, 0))); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| budget.release_text(1))).is_err() + ); +} +#[test] +fn duplicate_descriptions_refuse_before_a_second_grammar_decode() { + let text = "d".repeat(100); + for (body, extra, message) in [ + ( + format!("description \"{text}\"; description \"{text}\";"), + 108, + "Duplicate project description.", + ), + ( + format!( + "calculation c \"C\" id \"00000000-0000-0000-0000-000000000002\" {{ description \"{text}\"; description \"{text}\"; }}" + ), + 110, + "Duplicate field 'description'.", + ), + ] { + let b = bundle(&body, "Project"); + // Independently counted C is source plus the two12-byte owned paths; + // first description needs108/110, a second decoded copy would not fit. + let bytes = b.files["main.arcadia"].len() + 24 + extra; + let owner = held(b, 1000, bytes); + let e = failed(parse_text_producers(owner.input())); + assert_eq!(e.diagnostic.message, message); + assert_eq!( + failed(parse_carried(owner.input())).diagnostic.message, + message + ); + } +} diff --git a/crates/oa-core/src/language/analysis_source/mixed/project.rs b/crates/oa-core/src/language/analysis_source/mixed/project.rs index c695cb59..ba23e829 100644 --- a/crates/oa-core/src/language/analysis_source/mixed/project.rs +++ b/crates/oa-core/src/language/analysis_source/mixed/project.rs @@ -11,6 +11,17 @@ use crate::{Id, Kind, Project, RelationKind, analysis, language, validation}; use std::collections::BTreeMap; pub(in crate::language::analysis_source) mod accounting; use accounting::{Ledger, Slots, add}; +#[cfg(test)] +pub(super) fn text_producer_inventory(syntax: &ParsedMixedSyntax<'_>) -> Result { + let mut ledger = Ledger { + live: syntax + .carried + .as_ref() + .map_or_else(Slots::default, |g| g.slots()), + }; + accounting::syntax(syntax, &mut ledger)?; + Ok(ledger.live) +} pub(in crate::language::analysis_source) mod ordinary_index; /// Read-only authority: every component originates in the consumed syntax. diff --git a/crates/oa-core/src/language/analysis_source/mixed/project/accounting.rs b/crates/oa-core/src/language/analysis_source/mixed/project/accounting.rs index 7d6831f7..3e61e44f 100644 --- a/crates/oa-core/src/language/analysis_source/mixed/project/accounting.rs +++ b/crates/oa-core/src/language/analysis_source/mixed/project/accounting.rs @@ -104,9 +104,14 @@ struct Walk<'a, 'b> { budget: &'b Budget, work: Vec>, lower: LowerStorage, + text_total: Option, } impl<'a> Walk<'a, '_> { fn text(&mut self, text: &str, at: Span) -> Result<(), TypedError> { + if let Some(total) = &mut self.text_total { + *total = add(*total, text.len(), at)?; + return Ok(()); + } self.ledger .reserve(self.budget, Slots::text(text.len()), at) } @@ -236,6 +241,19 @@ pub(super) fn syntax( ledger: &mut Ledger, ) -> Result { let at = Span::default(); + let producer_text = syntax.budget.producer_text(); + if let Some((text, temporary)) = producer_text { + if temporary != 0 { + return Err(Failure::typed( + syntax, + None, + error(at, "Unreleased syntax text temporary."), + )); + } + ledger + .reserve(&syntax.budget, Slots::text(text), at) + .map_err(|e| Failure::typed(syntax, None, e))?; + } ledger .reserve( &syntax.budget, @@ -251,6 +269,7 @@ pub(super) fn syntax( budget: &syntax.budget, work: Vec::new(), lower: LowerStorage::default(), + text_total: producer_text.map(|_| 0), }; for (file, unit) in syntax.files.iter().enumerate() { let mut count = || -> Result<(), TypedError> { @@ -348,5 +367,12 @@ pub(super) fn syntax( }; count().map_err(|e| Failure::typed(syntax, Some(file), e))?; } + if walk.text_total != producer_text.map(|(text, _)| text) { + return Err(Failure::typed( + syntax, + None, + error(at, "Producer text differs from retained syntax inventory."), + )); + } Ok(walk.lower) } diff --git a/crates/oa-core/src/language/raw_tokens.rs b/crates/oa-core/src/language/raw_tokens.rs index c977b752..d894ff4e 100644 --- a/crates/oa-core/src/language/raw_tokens.rs +++ b/crates/oa-core/src/language/raw_tokens.rs @@ -32,14 +32,31 @@ impl<'a> Scanner<'a> { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum StringEvent { + Reserve { span: Span, payload: usize }, + Released { payload: usize }, +} impl Iterator for Scanner<'_> { type Item = Token; - fn next(&mut self) -> Option { + match self.next_with_strings(|_| Ok::<(), std::convert::Infallible>(())) { + Ok(token) => token, + Err(never) => match never {}, + } + } +} +impl Scanner<'_> { + pub(super) fn next_with_strings( + &mut self, + mut hook: impl FnMut(StringEvent) -> Result<(), E>, + ) -> Result, E> { let source = self.source; let bytes = source.as_bytes(); let start = self.cursor; - let first = *bytes.get(start)?; + let Some(&first) = bytes.get(start) else { + return Ok(None); + }; let mut legacy_number_part = false; let class = if matches!(first, b' ' | b'\t' | b'\r' | b'\n') { self.cursor += 1; @@ -123,11 +140,24 @@ impl Iterator for Scanner<'_> { break; } } - if closed && serde_json::from_str::(&source[start..self.cursor]).is_ok() { - Class::String + let valid = if closed { + let payload = self.cursor - start - 2; + hook(StringEvent::Reserve { + span: Span { + start, + end: self.cursor, + }, + payload, + })?; + // The temporary Result/String drops at this statement's end, + // before the refund hook runs. + let valid = serde_json::from_str::(&source[start..self.cursor]).is_ok(); + hook(StringEvent::Released { payload })?; + valid } else { - Class::Error - } + false + }; + if valid { Class::String } else { Class::Error } } else if b"{}();,[]:=@$+-*/<>!".contains(&first) { self.cursor += 1; if matches!(first, b'<' | b'>' | b'=' | b'!') && bytes.get(self.cursor) == Some(&b'=') { @@ -144,14 +174,14 @@ impl Iterator for Scanner<'_> { legacy_number_part = first == b'.'; Class::Error }; - Some(Token { + Ok(Some(Token { class, span: Span { start, end: self.cursor, }, legacy_number_part, - }) + })) } } diff --git a/harness/analysis-text-producer-implementation-20260925.md b/harness/analysis-text-producer-implementation-20260925.md new file mode 100644 index 00000000..f9f52efc --- /dev/null +++ b/harness/analysis-text-producer-implementation-20260925.md @@ -0,0 +1,26 @@ +# Private retained-syntax text producer prerequisite + +2026-09-25. Candidate on source baseline `8d52dab97c92cc31baae9f20d8c324b55818aa5d`, selected root contract `harness/analysis-text-producer-contract-20260925.md` at local-only `17836587`. Root publication is parent-owned and server capacity is not reported recovered. This note records authored work, not compiler/runtime acceptance. + +An explicit fixture-only `parse_text_producers` route derives bundle, guard and exact policies from CarriedInput. Existing legacy and carried-phase routes pass disabled mode. Budget adds only optional scalar carried/owned/temporary text counts; source/token/entry/reference/numeric Usage remains separate. Named storage checks use the inherited byte ceiling before copy/decode and preserve failed-reservation totals. Quoted text reserves raw payload P, reconciles decoded D<=P, and transfers the existing reservation continuously into owned text without a second fallible reservation. The conservative escaped-fit refusal is intentional only for this new route. + +One shared raw scanner body has a bounded before-validation/refund hook. Its original Iterator API uses a no-op hook. A completed quoted token reserves before the existing serde validation; the temporary result String drops at the statement end before refund. Unclosed tokens skip decoding. New-mode storage refusal may precede malformed escape classification; old routes retain that classification. There is no new decoder or numeric conversion. + +Typed and ordinary retained names, quoted fields, AST words/references/keys and projected key/reference copies reserve before creation. All three consumed UUID spelling sites drop then refund: ordinary optional IDs, typed identities, and ordinary HistoricalUuid. Nested field originals provisionally count as owned text alongside retained clones, then explicitly drop/refund; top-level fields and field_spans retain both copies. The declaration-kind spelling is borrowed from the original source. Existing duplicate description checks precede replacement/second grammar decoding. + +Successful syntax has no outstanding decoder/scanner temporary reservation. Resolver seeds C once, transfers producer T once, and changes the existing complete text walk into an independent checked sum/equality check for this mode. Existing routes keep progressive text reservation. A cfg(test), read-only project inventory helper invokes that same bounded walk on syntax-only fixtures; it cannot replace bundle/policy/baseline authority. + +Scope remains logical retained String lengths and named result temporaries. Decoder internal scratch/capacity, existing duplicate-name coordination copies, traversal/entry/metadata producer timing, model conversion/validation, Exact internals, allocator overhead and native stack retain their explicit separate domains. No PendingRename extraction, emitted-output parser transition, accepted result, public source8, persistence or history operation is added. + +## Six focused authored tests + +All new names are under `mixed::carried::text_producer`, selected by filter `text_producer`. Independent literal/count expectations came from the source-only text-producer oracle, not application outputs. + +- Empty real owner E complete cap146/145, finalT8; escaped project cap491/490, finalT41; independent syntax inventory, complete expected Project and Usage parity, and unchanged carried-phase112 control. +- Three-file escaped include cap358/357, C253/T45, final resolved298 and entry25. A separately labeled scanner-only state replay distinguishes scanner reservation from the identical later grammar threshold. +- Projection-dominant syntax cap304/303, finalT83 and independent inventory304; nested/reference/all-three-UUID syntax fixture cap453/452, finalT49/inventory432. The arbitrary payload fixture is explicitly not asserted to admit a Project. +- Raw/escaped Unicode, simple escapes, surrogate pairs and empty strings: rawP cap/refund and scanner hook order; malformed escapes/surrogates, unclosed tokens and explicit legacy error precedence. Arithmetic baselines are labeled separately from real-owner provenance. +- Explicit zero-C stronger route OA/SA complete model, warnings, final retained storage, Usage and numeric-work parity; checked overflow/refund fault injections. +- Project and typed duplicate descriptions with enough room for the first grammar decode but not a second, asserting the existing duplicate errors rather than a false successful overwrite. + +No old test or worker-stack allowance changed. Implementer ran no Cargo, test executable or rustfmt; parent owns formatting, compiler, prologue/headroom and runtime gates. Independent production review found no initial semantic blocker; final review binds parent-formatted source and authored tests before commit. Preserve all prior recorded failures and compare actual full-suite dispositions rather than claiming they are unchanged from source inspection.