//! Bounded experiment: preserve authored bytes while writing model deltas. //! This span index is not a production concrete syntax tree or formatter. use crate::{Entry, Parser, Record, compile}; use oa_core::{ Id, Kind, Project, language::{self, TokenClass}, source_project::SourceBundle, }; use std::{ collections::{BTreeMap, BTreeSet}, ops::Range, }; #[derive(Clone, Debug)] struct Token { text: String, span: Range, } #[derive(Clone, Debug)] struct Field { name: Range, value: Range, } #[derive(Clone, Debug)] enum Syntax { Raw, Port { keyword: Range, items: Range, }, Exchange { source: Range, target: Range, items: Range, }, } #[derive(Clone, Debug)] struct Site { path: String, span: Range, alias: Range, name: Option>, body: Option>, steps: Option>, fields: BTreeMap, syntax: Syntax, lexical_owner: Option, } #[derive(Clone, Debug)] struct Edit { path: String, span: Range, text: String, } fn tokens(source: &str, range: Range) -> Vec { language::tokenize(&source[range.clone()]) .into_iter() .filter(|token| !matches!(token.class, TokenClass::Comment | TokenClass::Whitespace)) .map(|token| Token { text: source[range.start + token.span.start..range.start + token.span.end].into(), span: range.start + token.span.start..range.start + token.span.end, }) .collect() } fn matching(tokens: &[Token], start: usize) -> Result { let mut stack = Vec::new(); for (i, token) in tokens.iter().enumerate().skip(start) { match token.text.as_str() { "{" => stack.push("}"), "[" => stack.push("]"), "}" | "]" => { stack .pop() .filter(|expected| *expected == token.text) .ok_or("Unbalanced token range")?; if stack.is_empty() { return Ok(i); } } _ => {} } if stack.len() > 64 { return Err("Index nesting limit".into()); } } Err("Unclosed token range".into()) } fn scalar_end(tokens: &[Token], start: usize) -> Result { let token = tokens.get(start).ok_or("Missing value")?; match token.text.as_str() { "{" | "[" => matching(tokens, start), "@" => { tokens.get(start + 1).ok_or("Missing reference alias")?; Ok(start + 1) } _ => Ok(start), } } fn index(bundle: &SourceBundle, model: &Project) -> Result, String> { let aliases: BTreeMap<_, _> = model .identities() .into_iter() .map(|(id, alias)| (alias, id)) .collect(); let mut sites = BTreeMap::new(); for (path, source) in &bundle.files { let ts = tokens(source, 0..source.len()); for i in 2..ts.len().saturating_sub(1) { if ts[i].text != "id" || !ts[i + 1].text.starts_with('"') { continue; } let named = ts[i - 1].text.starts_with('"'); let alias_at = if named { i - 2 } else { i - 1 }; if alias_at == 0 { continue; } let Some(&id) = aliases.get(ts[alias_at].text.as_str()) else { continue; }; let kind_at = alias_at - 1; let next = i + 2; let mut body = None; let syntax; let end; match ts.get(next).map(|t| t.text.as_str()) { Some("{") => { let close = matching(&ts, next)?; body = Some(ts[next].span.end..ts[close].span.start); syntax = Syntax::Raw; end = close; } Some("items") => { let last = scalar_end(&ts, next + 1)?; if ts.get(last + 1).is_none_or(|t| t.text != ";") { return Err("Invalid port shorthand terminator".into()); } syntax = Syntax::Port { keyword: ts[kind_at].span.clone(), items: ts[next + 1].span.start..ts[last].span.end, }; end = last + 1; } Some("from") => { if ts.get(next + 3).is_none_or(|t| t.text != "to") || ts.get(next + 6).is_none_or(|t| t.text != "items") { return Err("Unexpected exchange shorthand".into()); } let last = scalar_end(&ts, next + 7)?; syntax = Syntax::Exchange { source: ts[next + 1].span.start..ts[next + 2].span.end, target: ts[next + 4].span.start..ts[next + 5].span.end, items: ts[next + 7].span.start..ts[last].span.end, }; end = if ts.get(last + 2).is_some_and(|t| t.text == "{") { let close = matching(&ts, last + 2)?; body = Some(ts[last + 2].span.end..ts[close].span.start); close } else { last + 1 }; } _ => continue, } if sites .insert( id, Site { path: path.clone(), span: ts[kind_at].span.start..ts[end].span.end, alias: ts[alias_at].span.clone(), name: named.then(|| ts[i - 1].span.clone()), body, steps: None, fields: BTreeMap::new(), syntax, lexical_owner: None, }, ) .is_some() { return Err("Duplicate indexed identity".into()); } } } let spans = sites.clone(); for (&id, site) in &mut sites { site.lexical_owner = spans .iter() .filter(|(other, parent)| { **other != id && parent.path == site.path && parent.body.as_ref().is_some_and(|body| { body.start <= site.span.start && body.end >= site.span.end }) }) .min_by_key(|(_, parent)| parent.span.len()) .map(|(&id, _)| id); site.fields.clear(); if let Some(body) = &site.body { let ts = tokens(&bundle.files[&site.path], body.clone()); let mut at = 0; while at < ts.len() { if let Some(child) = spans .values() .find(|child| child.path == site.path && child.span.start == ts[at].span.start) { at = ts.partition_point(|token| token.span.start < child.span.end); continue; } let name = ts[at].text.clone(); // A project include is not a semantic model field. let last = scalar_end(&ts, at + 1)?; if ts.get(last + 1).is_none_or(|token| token.text != ";") { return Err(format!("Cannot index field {name}")); } if name == "steps" && ts[at + 1].text == "{" { site.steps = Some(ts[at + 1].span.end..ts[last].span.start); } else if name != "include" { site.fields.insert( name, Field { name: ts[at].span.clone(), value: ts[at + 1].span.start..ts[last].span.end, }, ); } at = last + 2; } } } if sites.len() != model.identities().len() { return Err("Not every semantic record has an indexed explicit header".into()); } Ok(sites) } fn canonical_records(model: &Project) -> Result, String> { let source = language::format(model); let aliases: BTreeMap<_, _> = model .identities() .into_iter() .map(|(id, alias)| (alias, id)) .collect(); Parser::new(&source)? .unit(true)? .into_iter() .filter_map(|entry| match entry { Entry::Record(record) => Some(record), _ => None, }) .map(|record| { let id = *aliases .get(record.alias.as_str()) .ok_or("Unknown canonical record")?; Ok((id, record)) }) .collect() } fn push(edits: &mut Vec, path: &str, span: Range, text: impl Into) { edits.push(Edit { path: path.into(), span, text: text.into(), }); } fn comments(source: &str, range: Range) -> bool { language::tokenize(&source[range]) .iter() .any(|t| t.class == TokenClass::Comment) } fn object_fields( source: &str, range: Range, ) -> Result>, String> { let ts = tokens(source, range); if ts.first().is_none_or(|t| t.text != "{") { return Ok(None); } let mut fields = BTreeMap::new(); let mut at = 1; while at < ts.len() - 1 { let last = scalar_end(&ts, at + 1)?; if ts.get(last + 1).is_none_or(|t| t.text != ";") { return Err("Unsupported object field form".into()); } fields.insert( ts[at].text.clone(), Field { name: ts[at].span.clone(), value: ts[at + 1].span.start..ts[last].span.end, }, ); at = last + 2; } Ok(Some(fields)) } fn list_items(source: &str, range: Range) -> Result>>, String> { let ts = tokens(source, range); if ts.first().is_none_or(|t| t.text != "[") { return Ok(None); } let mut items = Vec::new(); let mut at = 1; while at < ts.len() - 1 { let last = scalar_end(&ts, at)?; items.push(ts[at].span.start..ts[last].span.end); at = last + 1; if at < ts.len() - 1 { if ts[at].text != "," { return Err("Unsupported list form".into()); } at += 1; } } Ok(Some(items)) } /// Patch matched value tokens, retaining their authored ordering and all trivia. /// Added/removed object keys or commented-list restructuring are deliberately refused. fn value_edits( edits: &mut Vec, path: &str, source: &str, range: Range, old: &str, new: &str, ) -> Result<(), String> { if old == new { return Ok(()); } let old_fields = object_fields(old, 0..old.len())?; let new_fields = object_fields(new, 0..new.len())?; if let (Some(before), Some(after), Some(actual)) = ( old_fields.as_ref(), new_fields.as_ref(), object_fields(source, range.clone())?, ) { if before.keys().eq(after.keys()) && before.keys().all(|key| actual.contains_key(key)) { for (name, field) in before { value_edits( edits, path, source, actual[name].value.clone(), &old[field.value.clone()], &new[after[name].value.clone()], )?; } return Ok(()); } if before.len() == 1 && after.len() == 1 && actual.len() == 1 { let (old_key, old_field) = before.first_key_value().unwrap(); let (new_key, new_field) = after.first_key_value().unwrap(); let field = actual.get(old_key).ok_or("Constructor spelling mismatch")?; push(edits, path, field.name.clone(), new_key); return value_edits( edits, path, source, field.value.clone(), &old[old_field.value.clone()], &new[new_field.value.clone()], ); } return Err("Unsupported object-key addition/removal; no source was changed".into()); } if let (Some(before), Some(after), Some(actual)) = ( list_items(old, 0..old.len())?, list_items(new, 0..new.len())?, list_items(source, range.clone())?, ) { if before.len() != actual.len() { return Err("Authored list shape mismatch".into()); } if after.len() >= before.len() { for (i, item) in before.iter().enumerate() { value_edits( edits, path, source, actual[i].clone(), &old[item.clone()], &new[after[i].clone()], )?; } if after.len() > before.len() { if let Some(last) = actual.last() { push(edits, path, last.end..last.end, ","); } let close = range.end - 1; let extra = after[before.len()..] .iter() .map(|item| &new[item.clone()]) .collect::>() .join(", "); push( edits, path, close..close, if source[range.clone()].contains('\n') { format!("\n {extra}\n") } else { format!(" {extra}") }, ); } return Ok(()); } if comments(source, range.clone()) { return Err( "Commented list removal requires explicit comment ownership; no source was changed" .into(), ); } } if comments(source, range.clone()) { return Err("Unsupported commented value shape change; no source was changed".into()); } push(edits, path, range, new); Ok(()) } fn line_start(source: &str, at: usize) -> usize { source[..at].rfind('\n').map_or(0, |at| at + 1) } fn indent(source: &str, at: usize) -> Result { let prefix = &source[line_start(source, at)..at]; if prefix.chars().all(char::is_whitespace) { Ok(prefix.into()) } else { Err("Structural edits require declarations on their own lines".into()) } } fn envelope(source: &str, span: Range) -> Result, String> { let indentation = indent(source, span.start)?; let mut start = line_start(source, span.start); while start > 0 { let previous_end = start - 1; let previous_start = line_start(source, previous_end); let previous = &source[previous_start..previous_end]; if previous.starts_with(&format!("{indentation}//")) { start = previous_start } else { break; } } let end = source[span.end..] .find('\n') .map_or(source.len(), |n| span.end + n + 1); let suffix = source[span.end..end].trim(); if !suffix.is_empty() && !suffix.starts_with("//") { return Err("Structural edits require a whole-line declaration boundary".into()); } Ok(start..end) } fn apply_local( source: &str, range: Range, path: &str, edits: &[Edit], ) -> Result { let selected: Vec<_> = edits .iter() .filter(|edit| { edit.path == path && edit.span.start >= range.start && edit.span.end <= range.end }) .cloned() .collect(); let mut text = source[range.clone()].to_owned(); let mut selected = selected; selected.sort_by_key(|edit| (edit.span.start, edit.span.end)); let mut end = range.start; for edit in &selected { if edit.span.start < end { return Err("Overlapping source patches".into()); } end = edit.span.end; } for edit in selected.into_iter().rev() { text.replace_range( edit.span.start - range.start..edit.span.end - range.start, &edit.text, ); } Ok(text) } fn apply(bundle: &SourceBundle, edits: &[Edit]) -> Result { let mut output = bundle.clone(); for (path, source) in &bundle.files { output.files.insert( path.clone(), apply_local(source, 0..source.len(), path, edits)?, ); } Ok(output) } fn reindent(text: &str, old: &str, new: &str) -> String { text.split_inclusive('\n') .map(|line| { if line.trim().is_empty() { line.into() } else { format!("{new}{}", line.strip_prefix(old).unwrap_or(line)) } }) .collect() } fn insertion_at(source: &str, close: usize, text: &str) -> Edit { let start = line_start(source, close); let prefix = &source[start..close]; if prefix.chars().all(char::is_whitespace) { Edit { path: String::new(), span: start..start, text: text.into(), } } else { Edit { path: String::new(), span: close..close, text: format!( "\n{text}{}", source[start..] .chars() .take_while(|ch| *ch == ' ' || *ch == '\t') .collect::() ), } } } fn add_field( edits: &mut Vec, site: &Site, source: &str, name: &str, value: &str, ) -> Result<(), String> { let body = site .body .as_ref() .ok_or("This shorthand has no field escape body")?; let indentation = indent(source, site.span.start)?; let mut edit = insertion_at( source, body.end, &format!("{indentation} {name} {value};\n"), ); edit.path = site.path.clone(); edits.push(edit); Ok(()) } fn absorb(edits: &mut Vec, path: &str, range: Range) -> Result<(), String> { if edits.iter().any(|edit| { edit.path == path && edit.span.start < range.end && edit.span.end > range.start && !(edit.span.start >= range.start && edit.span.end <= range.end) }) { return Err("Overlapping structural/scalar edits are outside the experiment".into()); } edits.retain(|edit| { !(edit.path == path && edit.span.start >= range.start && edit.span.end <= range.end) }); Ok(()) } /// Write only deltas representable by the bounded experiment. Failure returns no /// replacement bundle; the caller's model and all input documents remain intact. pub fn writeback(bundle: &SourceBundle, candidate: &Project) -> Result { let before = compile(bundle)?; language::parse(&language::format(candidate)) .map_err(|e| format!("Invalid candidate: {e:?}"))?; if before.id != candidate.id || before.schema_version != candidate.schema_version { return Err("Project identity/schema replacement is outside the experiment".into()); } let sites = index(bundle, &before)?; // Rename aliases as one simultaneous lexical phase before other model deltas. // Re-index its result so alias cycles and changed token widths cannot cause // double rewrites or turn an unchanged parent identity into a semantic move. let candidate_aliases: BTreeMap<_, _> = candidate.identities().into_iter().collect(); let renamed: BTreeMap<_, _> = before .identities() .into_iter() .filter_map(|(id, old)| { candidate_aliases .get(&id) .filter(|new| **new != old) .map(|new| (old.to_owned(), (*new).to_owned())) }) .collect(); if !renamed.is_empty() { let mut alias_edits = Vec::new(); for (id, old) in before.identities() { if let Some(new) = renamed.get(old) { let site = &sites[&id]; push(&mut alias_edits, &site.path, site.alias.clone(), new); } } for (path, source) in &bundle.files { let ts = tokens(source, 0..source.len()); for pair in ts.windows(2) { if pair[0].text == "@" && let Some(new) = renamed.get(&pair[1].text) { push(&mut alias_edits, path, pair[1].span.clone(), new); } } } return writeback(&apply(bundle, &alias_edits)?, candidate); } let old = canonical_records(&before)?; let new = canonical_records(candidate)?; if old.keys().any(|id| !new.contains_key(id)) { return Err("Record deletion is outside the experiment".into()); } let new_ids: BTreeSet<_> = new .keys() .filter(|id| !old.contains_key(id)) .copied() .collect(); if new_ids.iter().any(|id| { candidate .elements .get(id) .is_none_or(|e| e.kind() != Kind::Occurrence) }) { return Err("Only occurrence creation is in the experiment".into()); } let mut edits = Vec::new(); let mut moves = Vec::new(); let mut orders = Vec::new(); for (id, record) in &old { let next = &new[id]; let site = &sites[id]; let source = &bundle.files[&site.path]; if record.kind != next.kind { return Err("Declaration kind changes are outside the experiment".into()); } if let Some(name) = &site.name { let old_header = tokens(&record.header, 0..record.header.len()); let new_header = tokens(&next.header, 0..next.header.len()); if old_header[2].text != new_header[2].text { push(&mut edits, &site.path, name.clone(), &new_header[2].text); } } for (field, value) in &next.fields { let previous = record .fields .get(field) .ok_or("Added constructor field is outside the experiment")?; if previous == value { continue; } if field == "parent" && let Some(element) = before.elements.get(id) && site.lexical_owner == element.parent && element.parent.is_some() && element.parent != candidate.elements[id].parent { moves.push((*id, candidate.elements[id].parent)); if !site.fields.contains_key("parent") { continue; } } if field == "steps" && site.steps.is_some() { orders.push(*id); continue; } match (&site.syntax, field.as_str()) { (Syntax::Port { keyword, items }, "contract") => { let e = &candidate.elements[id]; let Some(oa_core::SharedBody::Port { contract, .. }) = e.shared() else { return Err("Expected port contract".into()); }; let (word, list) = match contract { oa_core::PortContract::FunctionInput { incoming_items } => { ("input", incoming_items) } oa_core::PortContract::FunctionOutput { outgoing_items } => { ("output", outgoing_items) } _ => { return Err( "Only typed function input/output shorthand is supported".into() ); } }; push(&mut edits, &site.path, keyword.clone(), word); let ids: BTreeMap<_, _> = candidate.identities().into_iter().collect(); let value = format!( "[{}]", list.iter() .map(|id| format!("@{}", ids[id])) .collect::>() .join(", ") ); // The source list is already semantically the previous port item list. let previous = &source[items.clone()]; value_edits( &mut edits, &site.path, source, items.clone(), previous, &value, )?; } (Syntax::Exchange { source: span, .. }, "source_port") | (Syntax::Exchange { target: span, .. }, "target_port") => { push(&mut edits, &site.path, span.clone(), value) } (Syntax::Exchange { .. }, "source" | "target") => {} (Syntax::Exchange { items, .. }, "items") => value_edits( &mut edits, &site.path, source, items.clone(), previous, value, )?, _ => { if let Some(origin) = site.fields.get(field) { value_edits( &mut edits, &site.path, source, origin.value.clone(), previous, value, )?; } else { add_field(&mut edits, site, source, field, value)?; } } } } } let project_site = &sites[&before.id]; if before.name != candidate.name { push( &mut edits, &project_site.path, project_site.name.clone().unwrap(), serde_json::to_string(&candidate.name).unwrap(), ); } if before.alias != candidate.alias { push( &mut edits, &project_site.path, project_site.alias.clone(), &candidate.alias, ); } if before.description != candidate.description { let value = serde_json::to_string(&candidate.description).unwrap(); if let Some(field) = project_site.fields.get("description") { push(&mut edits, &project_site.path, field.value.clone(), value); } else { add_field( &mut edits, project_site, &bundle.files[&project_site.path], "description", &value, )?; } } let mut handled_new = BTreeSet::new(); for id in orders { let site = &sites[&id]; let range = site.steps.clone().unwrap(); let source = &bundle.files[&site.path]; let previous = before.elements[&id] .steps() .ok_or("Ordered owner missing")?; let next = candidate.elements[&id] .steps() .ok_or("Ordered owner missing")?; if previous.iter().any(|id| !next.contains(id)) { return Err("Removing an ordered member without deleting it needs a separate authoring decision".into()); } let mut blocks = BTreeMap::new(); let mut sorted = Vec::new(); for child in previous { let child_site = &sites[child]; if child_site.path != site.path { return Err("Ordered blocks cannot span included files".into()); } let env = envelope(source, child_site.span.clone())?; if env.start < range.start || env.end > range.end { return Err("Ordered member is not wholly inside its steps block".into()); } blocks.insert( *child, apply_local(source, env.clone(), &site.path, &edits)?, ); sorted.push(env); } sorted.sort_by_key(|span| span.start); for pair in sorted.windows(2) { if !source[pair[0].end..pair[1].start].trim().is_empty() { return Err("Unattached comments between ordered blocks need explicit ownership before reordering".into()); } } let prefix_end = sorted.first().map_or(range.end, |span| span.start); let suffix_start = sorted.last().map_or(range.end, |span| span.end); let indentation = format!("{} ", indent(source, site.span.start)?); let mut content = source[range.start..prefix_end].to_owned(); for child in next { if let Some(block) = blocks.get(child) { content.push_str(block); } else if new_ids.contains(child) && candidate.elements[child].parent == Some(id) { let record = &new[child]; let mut block = format!("{indentation}{} {{\n", record.header); for (field, value) in &record.fields { if field == "parent" || (["description", "precondition", "postcondition"] .contains(&field.as_str()) && value == "\"\"") { continue; } block.push_str(&format!("{indentation} {field} {value};\n")); } block.push_str(&format!("{indentation}}}\n")); content.push_str(&block); handled_new.insert(*child); } else { return Err( "Moving existing occurrences between blocks is outside the experiment".into(), ); } } content.push_str(&source[suffix_start..range.end]); absorb(&mut edits, &site.path, range.clone())?; push(&mut edits, &site.path, range, content); } if handled_new != new_ids { return Err("A new occurrence must be inserted into an existing B steps block".into()); } let mut moved_ranges = Vec::new(); for (id, parent) in moves { let site = &sites[&id]; let source = &bundle.files[&site.path]; let range = envelope(source, site.span.clone())?; if moved_ranges .iter() .any(|(path, other): &(String, Range)| { path == &site.path && range.start < other.end && range.end > other.start }) { return Err("Nested simultaneous ownership moves are outside the experiment".into()); } moved_ranges.push((site.path.clone(), range.clone())); let target = &sites[&parent.unwrap_or(before.id)]; let target_source = &bundle.files[&target.path]; let body = target.body.as_ref().ok_or("New owner has no body")?; let old_indent = indent(source, site.span.start)?; let new_indent = if parent.is_some() { format!("{} ", indent(target_source, target.span.start)?) } else { " ".into() }; let block = reindent( &apply_local(source, range.clone(), &site.path, &edits)?, &old_indent, &new_indent, ); absorb(&mut edits, &site.path, range.clone())?; push(&mut edits, &site.path, range, ""); let mut insertion = insertion_at(target_source, body.end, &block); insertion.path = target.path.clone(); edits.push(insertion); } let output = apply(bundle, &edits)?; if compile(&output)? != *candidate { return Err( "Writeback did not reproduce the requested model; no source was changed".into(), ); } Ok(output) }