From 06b33848cea5a0b1a6168d79a68e5185c190383e Mon Sep 17 00:00:00 2001 From: "sorokin.oleg.edu" Date: Sat, 5 Sep 2026 19:14:05 +0300 Subject: [PATCH] Add pure modular source3 project compiler and update APIs --- crates/oa-core/src/language.rs | 355 ++++++-- crates/oa-core/src/lib.rs | 1 + crates/oa-core/src/source_project.rs | 1091 ++++++++++++++++++++++++ crates/oa-core/tests/source_project.rs | 688 +++++++++++++++ docs/language.rst | 122 ++- harness/release-source-project.md | 56 ++ 6 files changed, 2219 insertions(+), 94 deletions(-) create mode 100644 crates/oa-core/src/source_project.rs create mode 100644 crates/oa-core/tests/source_project.rs create mode 100644 harness/release-source-project.md diff --git a/crates/oa-core/src/language.rs b/crates/oa-core/src/language.rs index 18083cd3..c7bd274b 100644 --- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -219,12 +219,12 @@ impl fmt::Display for Diagnostic { } #[derive(Debug)] -struct Error { - code: &'static str, - offset: usize, - message: String, +pub(crate) struct Error { + pub(crate) code: &'static str, + pub(crate) offset: usize, + pub(crate) message: String, } -fn diagnostic(source: &str, error: Error) -> Diagnostic { +pub(crate) fn diagnostic(source: &str, error: Error) -> Diagnostic { let start = error.offset.min(source.len()); let prefix = &source[..start]; Diagnostic { @@ -244,7 +244,7 @@ fn diagnostic(source: &str, error: Error) -> Diagnostic { fn reserved(word: &str) -> bool { crate::reserved_alias(word) || declaration_kind(word).is_some() - || matches!(word, "none" | "node") + || matches!(word, "none" | "node" | "include") } #[derive(Debug, Clone, Copy)] @@ -534,11 +534,6 @@ fn offset_for(source: &str, line: usize, column: usize) -> usize { /// Canonical output uses persistent aliases and explicit identities. Typed /// nested records are ordinary language fields; JSON documents are not embedded. pub fn format(project: &Project) -> String { - let aliases = project - .identities() - .into_iter() - .map(|(id, a)| (id, a.to_owned())) - .collect::>(); let mut out = std::format!( "arcadia 3;\nproject {} {} id \"{}\" {{\n description {};\n\n", project.alias, @@ -546,7 +541,23 @@ pub fn format(project: &Project) -> String { project.id, quote(&project.description) ); + for (_, declaration) in format_declarations(project) { + out.push_str(&declaration); + } + out.push_str("}\n"); + out +} + +/// The same typed declaration writer is used by flat and modular source output. +pub(crate) fn format_declarations(project: &Project) -> Vec<(Id, String)> { + let aliases = project + .identities() + .into_iter() + .map(|(id, alias)| (id, alias.to_owned())) + .collect(); + let mut declarations = Vec::new(); for e in project.elements.values() { + let mut out = String::new(); let mut fields = payload(&serde_json::to_value(&e.body).unwrap()); fields.insert("description".into(), Value::String(e.description.clone())); fields.insert("parent".into(), serde_json::to_value(e.parent).unwrap()); @@ -559,8 +570,10 @@ pub fn format(project: &Project) -> String { &fields, &aliases, ); + declarations.push((e.id, out)); } for r in project.relations.values() { + let mut out = String::new(); let mut fields = payload(&serde_json::to_value(&r.body).unwrap()); fields.insert("description".into(), Value::String(r.description.clone())); write_declaration( @@ -572,8 +585,10 @@ pub fn format(project: &Project) -> String { &fields, &aliases, ); + declarations.push((r.id, out)); } for d in project.diagrams.values() { + let mut out = String::new(); let mut fields = record_fields(d); fields.remove("nodes"); write_declaration( @@ -585,13 +600,17 @@ pub fn format(project: &Project) -> String { &fields, &aliases, ); + declarations.push((d.id, out)); for n in d.nodes.values() { + let mut out = String::new(); let mut fields = record_fields(n); fields.insert("diagram".into(), Value::String(d.id.to_string())); write_declaration(&mut out, "node", &n.alias, None, n.id, &fields, &aliases); + declarations.push((n.id, out)); } } for a in project.artifact_links.values() { + let mut out = String::new(); write_declaration( &mut out, "artifact", @@ -601,8 +620,10 @@ pub fn format(project: &Project) -> String { &record_fields(a), &aliases, ); + declarations.push((a.id, out)); } for t in project.traces.values() { + let mut out = String::new(); write_declaration( &mut out, "trace", @@ -612,10 +633,11 @@ pub fn format(project: &Project) -> String { &record_fields(t), &aliases, ); + declarations.push((t.id, out)); } - out.push_str("}\n"); - out + declarations } + fn record_fields(value: &T) -> Map { let mut fields = serde_json::to_value(value) .unwrap() @@ -867,13 +889,49 @@ fn strict_tokens(source: &str) -> Result, Error> { }); Ok(out) } -struct Declaration { +pub(crate) struct Declaration { kind: DeclKind, - alias: String, + pub(crate) alias: String, name: String, - id: Id, + pub(crate) id: Id, fields: BTreeMap, offset: usize, + pub(crate) span: Span, + pub(crate) id_insertion: Option, + pub(crate) field_spans: BTreeMap, +} +impl Declaration { + pub(crate) fn is_graph_object(&self) -> bool { + matches!(self.kind, DeclKind::Element(..) | DeclKind::Relation(..)) + } +} +pub(crate) struct SourceHeader { + pub(crate) project: Project, + pub(crate) span: Span, + pub(crate) id_insertion: Option, + pub(crate) description: Option, + pub(crate) body: Span, +} +pub(crate) struct SourceUnit { + pub(crate) header: Option, + pub(crate) declarations: Vec, + pub(crate) includes: Vec<(String, Span)>, +} +pub(crate) fn parse_unit( + source: &str, + entry: bool, + limits: ParseLimits, +) -> Result> { + let tokens = strict_tokens(source).map_err(|e| Box::new(diagnostic(source, e)))?; + Parser { + source, + tokens, + cursor: 0, + value_depth: 0, + limits, + } + .unit(entry, true) + .map_err(|e| Box::new(diagnostic(source, e))) } struct Parser<'a> { source: &'a str, @@ -1029,59 +1087,140 @@ impl Parser<'_> { } } fn document(&mut self) -> Result { - self.take("arcadia")?; - if self.text() != "3" { - return Err(self.error("Unsupported schema version: expected 'arcadia 3;'.")); + let unit = self.unit(true, false)?; + let header = unit.header.expect("entry header"); + let mut aliases = BTreeMap::from([(header.project.alias.clone(), header.project.id)]); + let mut locations = BTreeMap::from([(header.project.id, header.span.start)]); + let mut fields = BTreeMap::new(); + for d in &unit.declarations { + if aliases.insert(d.alias.clone(), d.id).is_some() { + return Err(Error { + code: "invalid_source", + offset: d.offset, + message: format!("Duplicate alias '{}'.", d.alias), + }); + } + if locations.insert(d.id, d.offset).is_some() { + return Err(Error { + code: "invalid_source", + offset: d.offset, + message: format!("Duplicate identity '{}'.", d.id), + }); + } + fields.extend( + d.field_spans + .iter() + .map(|(key, span)| ((d.id, key.clone()), span.start)), + ); } - self.cursor += 1; - self.take(";")?; - self.take("project")?; - let alias = self.identifier()?; - let name = self.quoted()?; - let id = self.optional_id()?; - self.take("{")?; - let mut project = Project { - schema_version: SCHEMA_VERSION, - id, - alias: alias.clone(), - name, - description: String::new(), - elements: BTreeMap::new(), - relations: BTreeMap::new(), - diagrams: BTreeMap::new(), - artifact_links: BTreeMap::new(), - traces: BTreeMap::new(), + let project = + resolve_declarations(header.project, &unit.declarations, &aliases, self.limits) + .map_err(|(_, error)| error)?; + Ok(ParsedDocument { + project, + migration: None, + locations, + fields, + }) + } + fn unit(&mut self, entry: bool, includes_allowed: bool) -> Result { + let mut header = if entry { + self.take("arcadia")?; + if self.text() != "3" { + return Err(self.error("Unsupported schema version: expected 'arcadia 3;'.")); + } + self.cursor += 1; + self.take(";")?; + let start = self.current().span.start; + self.take("project")?; + let alias = self.identifier()?; + let name = self.quoted()?; + let id_insertion = (self.text() != "id").then_some(self.current().span.start); + let id = self.optional_id()?; + let end = self.current().span.start; + self.take("{")?; + Some(SourceHeader { + project: Project { + schema_version: SCHEMA_VERSION, + id, + alias, + name, + description: String::new(), + elements: BTreeMap::new(), + relations: BTreeMap::new(), + diagrams: BTreeMap::new(), + artifact_links: BTreeMap::new(), + traces: BTreeMap::new(), + }, + span: Span { start, end }, + id_insertion, + description: None, + body: Span { + start: end + 1, + end: end + 1, + }, + }) + } else { + None }; - let mut aliases = BTreeMap::from([(alias, id)]); - let mut locations = BTreeMap::from([(id, 0)]); let mut declarations = Vec::new(); + let mut includes = Vec::new(); let mut graph_count = 0usize; - let mut field_locations = BTreeMap::new(); - let mut has_description = false; - while self.text() != "}" { - if self.text() == "description" { - if has_description { - return Err(self.error("Duplicate project description.")); + while if entry { + self.text() != "}" + } else { + !self.text().is_empty() + } { + let offset = self.current().span.start; + if self.text() == "include" { + if !includes_allowed { + return Err(self.error("An include requires a source bundle; single-string parsing never reads files.")); } - has_description = true; self.cursor += 1; - project.description = self.quoted()?; + let path = self.quoted()?; self.take(";")?; + includes.push(( + path, + Span { + start: offset, + end: self.tokens[self.cursor - 1].span.end, + }, + )); + continue; + } + if self.text() == "description" + && let Some(mut entry_header) = header.take() + { + if entry_header.description.is_some() { + return Err(self.error("Duplicate project description.")); + } + self.cursor += 1; + let description = self.quoted()?; + self.take(";")?; + entry_header.project.description = description; + entry_header.description = Some(Span { + start: offset, + end: self.tokens[self.cursor - 1].span.end, + }); + header = Some(entry_header); continue; } - let offset = self.current().span.start; let word = self.text().to_owned(); let kind = declaration_kind(&word) - .ok_or_else(|| self.error(std::format!("Unknown declaration '{word}'.")))?; + .ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?; graph_count += usize::from(matches!( kind, DeclKind::Element(..) | DeclKind::Relation(..) )); self.limits - .check_counts(graph_count, declarations.len().saturating_add(2), 0) + .check_counts( + graph_count, + declarations.len().saturating_add(1 + usize::from(entry)), + 0, + ) .map_err(|message| Error { code: "model_limit", - offset: self.current().span.start, + offset, message, })?; self.cursor += 1; @@ -1091,25 +1230,29 @@ impl Parser<'_> { } else { self.quoted()? }; + let id_insertion = (self.text() != "id").then_some(self.current().span.start); let id = self.optional_id()?; let fields = self.fields()?; - for (field, value) in &fields { - field_locations.insert((id, field.clone()), value.offset()); - } - if aliases.insert(alias.clone(), id).is_some() { - return Err(Error { - code: "invalid_source", - offset, - message: std::format!("Duplicate alias '{alias}'."), - }); - } - if locations.insert(id, offset).is_some() { - return Err(Error { - code: "invalid_source", - offset, - message: std::format!("Duplicate identity '{id}'."), - }); - } + let field_spans = fields + .iter() + .map(|(key, value)| { + let start = value.offset(); + let mut depth = 0usize; + let mut end = start; + for token in + self.tokens[self.tokens.partition_point(|t| t.span.start < start)..].iter() + { + match token.text(self.source) { + ";" if depth == 0 => break, + "{" | "[" => depth += 1, + "}" | "]" => depth = depth.saturating_sub(1), + _ => {} + } + end = token.span.end; + } + (key.clone(), Span { start, end }) + }) + .collect(); declarations.push(Declaration { kind, alias, @@ -1117,20 +1260,44 @@ impl Parser<'_> { id, fields, offset, + span: Span { + start: offset, + end: self.tokens[self.cursor - 1].span.end, + }, + id_insertion, + field_spans, }); } - self.take("}")?; + if entry { + header.as_mut().expect("entry header").body.end = self.current().span.start; + self.take("}")?; + } if !self.text().is_empty() { return Err(self.error("Unexpected content after the project closing brace.")); } - let mut nodes = Vec::new(); - for declaration in declarations { + Ok(SourceUnit { + header, + declarations, + includes, + }) + } +} + +pub(crate) fn resolve_declarations( + mut project: Project, + declarations: &[Declaration], + aliases: &BTreeMap, + limits: ParseLimits, +) -> Result { + let mut nodes = Vec::new(); + for (index, declaration) in declarations.iter().enumerate() { + (|| -> Result<(), Error> { let mut fields = declaration .fields .iter() - .map(|(k, v)| Ok((k.clone(), v.resolve(k, &aliases)?))) + .map(|(k, v)| Ok((k.clone(), v.resolve(k, aliases)?))) .collect::, Error>>()?; - let d = &declaration; + let d = declaration; let conversion_error = |message: String| Error { code: "invalid_source", offset: d @@ -1249,7 +1416,7 @@ impl Parser<'_> { header(&mut record, d, false); let node: NodePlacement = serde_json::from_value(Value::Object(record)) .map_err(|e| conversion_error(e.to_string()))?; - nodes.push((diagram, node, d.offset)); + nodes.push((diagram, node, d.offset, index)); } DeclKind::Artifact => { if !fields.contains_key("owner") { @@ -1280,28 +1447,34 @@ impl Parser<'_> { project.traces.insert(d.id, trace); } } - } - for (diagram, node, offset) in nodes { - let target = project.diagrams.get_mut(&diagram).ok_or_else(|| Error { + Ok(()) + })().map_err(|error| (index, error))?; + } + for (diagram, node, offset, index) in nodes { + let target = project + .diagrams + .get_mut(&diagram) + .ok_or_else(|| Error { code: "invalid_source", offset, message: "Node 'diagram' must reference a diagram declaration.".into(), - })?; - target.nodes.insert(node.id, node); - } - self.limits.check_model(&project).map_err(|message| Error { - code: "model_limit", - offset: 0, - message, - })?; - Ok(ParsedDocument { - project, - migration: None, - locations, - fields: field_locations, - }) + }) + .map_err(|error| (index, error))?; + target.nodes.insert(node.id, node); } + limits.check_model(&project).map_err(|message| { + ( + usize::MAX, + Error { + code: "model_limit", + offset: 0, + message, + }, + ) + })?; + Ok(project) } + fn header(fields: &mut Map, d: &Declaration, named: bool) { fields.insert("id".into(), Value::String(d.id.to_string())); fields.insert("alias".into(), Value::String(d.alias.clone())); diff --git a/crates/oa-core/src/lib.rs b/crates/oa-core/src/lib.rs index a03f5546..97a41dcd 100644 --- a/crates/oa-core/src/lib.rs +++ b/crates/oa-core/src/lib.rs @@ -6,6 +6,7 @@ pub mod ports; pub mod projection; pub mod references; pub mod session; +pub mod source_project; pub mod transitions; pub mod validation; pub use model::*; diff --git a/crates/oa-core/src/source_project.rs b/crates/oa-core/src/source_project.rs new file mode 100644 index 00000000..8d672fc7 --- /dev/null +++ b/crates/oa-core/src/source_project.rs @@ -0,0 +1,1091 @@ +//! Pure, bounded source3 projects. All includes resolve against supplied UTF-8 +//! documents; this module never reads a filesystem, URL, or process directory. +use crate::{Id, Kind, Project, language, persistence::DecodeResult, validation}; +use language::{Diagnostic, ParseLimits, Span}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// A portable slash-separated path, checked at every bundle boundary. +pub type LogicalPath = String; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceBundle { + pub entry: LogicalPath, + #[serde(deserialize_with = "unique_files")] + pub files: BTreeMap, +} + +fn unique_files<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct Files; + impl<'de> serde::de::Visitor<'de> for Files { + type Value = BTreeMap; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("unique logical source paths mapped to UTF-8 text") + } + fn visit_map>( + self, + mut access: A, + ) -> Result { + let mut files = BTreeMap::new(); + while let Some((path, source)) = access.next_entry::()? { + if files.insert(path.clone(), source).is_some() { + return Err(serde::de::Error::custom(format!( + "Duplicate source path '{path}'." + ))); + } + } + Ok(files) + } + } + deserializer.deserialize_map(Files) +} + +#[derive(Debug, Clone, Copy)] +pub struct ProjectLimits { + pub max_files: usize, + /// Number of include edges from the entry to a document; the entry is zero. + pub max_include_depth: usize, + pub max_include_edges: usize, + /// Existing language budgets, applied to the whole project, never per file. + pub model: ParseLimits, +} +impl Default for ProjectLimits { + fn default() -> Self { + Self { + max_files: 256, + max_include_depth: 32, + max_include_edges: 1_024, + model: ParseLimits { + max_source_bytes: 64 * 1024 * 1024, + ..ParseLimits::UNLIMITED + }, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceLocation { + pub path: LogicalPath, + pub span: Span, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceDiagnostic { + pub path: LogicalPath, + pub diagnostic: Diagnostic, + pub related: Vec, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IncludeLocation { + pub location: SourceLocation, + pub target: LogicalPath, +} + +/// Derived positions in `CompiledSourceProject::bundle`, including inserted IDs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceIndex { + /// Includes the project header itself under the project UUID. + pub declarations: BTreeMap, + /// Complete value spans for top-level declaration fields. + pub fields: BTreeMap<(Id, String), SourceLocation>, + pub includes: Vec, + pub project_header: SourceLocation, + pub project_body: SourceLocation, + /// The complete `description ...;` field, when explicitly present. + pub project_description: Option, +} + +#[derive(Debug, Clone)] +pub struct CompiledSourceProject { + /// Exact input text except for explicit UUIDs inserted into missing headers. + pub bundle: SourceBundle, + pub decoded: DecodeResult, + pub index: SourceIndex, + /// Located engineering warnings. Errors are returned instead of a candidate. + pub diagnostics: Vec, +} + +/// Normalize an include relative to its containing file. Bundle keys themselves +/// must already be canonical. Case collisions are rejected separately. +pub fn resolve_include(from: &str, include: &str) -> Result { + if !from.is_empty() && resolve_include("", from).as_deref() != Ok(from) { + return Err("The containing document path is not canonical.".into()); + } + if include.is_empty() + || include.starts_with('/') + || include.contains(['\\', ':', '<', '>', '"', '|', '?', '*']) + || include.chars().any(char::is_control) + { + return Err("Expected a relative portable source path.".into()); + } + let mut parts: Vec<&str> = from + .rsplit_once('/') + .map_or_else(Vec::new, |(dir, _)| dir.split('/').collect()); + for part in include.split('/') { + match part { + "" => return Err("Empty path segments are not allowed.".into()), + "." => {} + ".." => { + if parts.pop().is_none() { + return Err("Include escapes the source project root.".into()); + } + } + _ => { + let base = part.split('.').next().unwrap_or("").to_ascii_uppercase(); + if part.ends_with(['.', ' ']) + || matches!(base.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (base.len() == 4 + && (base.starts_with("COM") || base.starts_with("LPT")) + && matches!(base.as_bytes()[3], b'1'..=b'9')) + { + return Err("Path contains a non-portable file name.".into()); + } + parts.push(part); + } + } + } + let path = parts.join("/"); + if !path.ends_with(".arcadia") { + return Err("Source documents must have the .arcadia suffix.".into()); + } + Ok(path) +} + +fn problem( + bundle: &SourceBundle, + path: &str, + span: Span, + code: &'static str, + message: impl Into, +) -> SourceDiagnostic { + let source = bundle.files.get(path).map_or("", String::as_str); + located_problem(path, source, span, code, message) +} + +fn located_problem( + path: &str, + source: &str, + span: Span, + code: &'static str, + message: impl Into, +) -> SourceDiagnostic { + let mut diagnostic = language::diagnostic( + source, + language::Error { + code, + offset: span.start, + message: message.into(), + }, + ); + if span.end <= source.len() && source.is_char_boundary(span.end) { + diagnostic.span.end = span.end.max(diagnostic.span.start); + } + SourceDiagnostic { + path: path.into(), + diagnostic, + related: Vec::new(), + } +} + +/// Inspect one entry or fragment before its included documents are available. +/// Uses the same strict syntax parser as compilation, never a token/string scan. +/// A syntax error returns located diagnostics and no partial include list; +/// global references and engineering validity are checked by `compile_bundle`. +/// Callers must additionally enforce aggregate budgets while discovering files. +pub fn inspect_includes( + path: &str, + source: &str, + is_entry: bool, + limits: ProjectLimits, +) -> Result, Vec> { + let fail = |code, message| { + vec![located_problem( + path, + source, + Span::default(), + code, + message, + )] + }; + if resolve_include("", path).as_deref() != Ok(path) || (is_entry && path.contains('/')) { + return Err(fail( + "invalid_path", + "Non-canonical source document path.".into(), + )); + } + if limits.max_files == 0 { + return Err(fail( + "project_limit", + "The file admission budget is zero.".into(), + )); + } + if source.len() > limits.model.max_source_bytes { + return Err(fail( + "source_limit", + format!("Source exceeds {} bytes.", limits.model.max_source_bytes), + )); + } + let unit = language::parse_unit(source, is_entry, limits.model).map_err(|diagnostic| { + vec![SourceDiagnostic { + path: path.into(), + diagnostic: *diagnostic, + related: Vec::new(), + }] + })?; + let mut includes = Vec::new(); + for (raw, span) in unit.includes { + if includes.len() >= limits.max_include_edges { + return Err(vec![located_problem( + path, + source, + span, + "project_limit", + format!("Include graph exceeds {} edges.", limits.max_include_edges), + )]); + } + let target = resolve_include(path, &raw).map_err(|message| { + vec![located_problem(path, source, span, "invalid_path", message)] + })?; + includes.push(IncludeLocation { + location: SourceLocation { + path: path.into(), + span, + }, + target, + }); + } + Ok(includes) +} + +type Edits = BTreeMap>; + +/// Compile all reachable files as one source3 model and stabilize omitted UUIDs. +/// Missing/unreachable files and every admission or engineering error leave the +/// input untouched. V1/V2 single-file migration remains in `language::parse`. +pub fn compile_bundle( + bundle: &SourceBundle, + limits: ProjectLimits, +) -> Result> { + let (compiled, insertions) = compile_once(bundle, limits)?; + if insertions.is_empty() { + return Ok(compiled); + } + let mut stable = bundle.clone(); + apply_edits(&mut stable, insertions)?; + // Re-index the actual returned bytes. Never expose offsets into pre-ID text. + compile_once(&stable, limits).map(|(compiled, _)| compiled) +} + +fn compile_once( + bundle: &SourceBundle, + limits: ProjectLimits, +) -> Result<(CompiledSourceProject, Edits), Vec> { + let fail = |code, message| { + vec![problem( + bundle, + &bundle.entry, + Span::default(), + code, + message, + )] + }; + if bundle.entry.contains('/') { + return Err(fail( + "invalid_path", + "The entry file must be at the source project root.".into(), + )); + } + if bundle.files.len() > limits.max_files { + return Err(fail( + "project_limit", + format!("Source project exceeds {} files.", limits.max_files), + )); + } + let bytes = bundle + .files + .values() + .fold(0usize, |sum, text| sum.saturating_add(text.len())); + if bytes > limits.model.max_source_bytes { + return Err(fail( + "source_limit", + format!( + "Combined source exceeds {} bytes.", + limits.model.max_source_bytes + ), + )); + } + let mut folded = BTreeMap::new(); + for path in std::iter::once(&bundle.entry).chain(bundle.files.keys()) { + if resolve_include("", path).as_ref() != Ok(path) { + return Err(fail( + "invalid_path", + format!("Non-canonical source path '{path}'."), + )); + } + if let Some(previous) = folded.insert(path.to_lowercase(), path) + && previous != path + { + return Err(fail( + "duplicate_path", + format!("Case-colliding source paths '{previous}' and '{path}'."), + )); + } + } + if !bundle.files.contains_key(&bundle.entry) { + return Err(fail( + "missing_include", + "The entry file is missing from the source bundle.".into(), + )); + } + + enum Visit { + Enter(LogicalPath, usize, Option), + Leave, + } + let mut stack = vec![Visit::Enter(bundle.entry.clone(), 0, None)]; + let mut active: Vec = Vec::new(); + let mut visited = BTreeSet::new(); + let mut declarations = Vec::new(); + let mut declaration_paths = Vec::new(); + let mut aliases = BTreeMap::new(); + let mut alias_locations: BTreeMap = BTreeMap::new(); + let mut project_header = None; + let mut declaration_locations = BTreeMap::new(); + let mut fields = BTreeMap::new(); + let mut includes = Vec::new(); + let mut insertions: Edits = BTreeMap::new(); + let mut graph_count = 0usize; + while let Some(visit) = stack.pop() { + let Visit::Enter(path, depth, origin) = visit else { + active.pop(); + continue; + }; + let at = origin.unwrap_or(SourceLocation { + path: path.clone(), + span: Span::default(), + }); + if let Some(start) = active.iter().position(|file| file == &path) { + let chain = active[start..] + .iter() + .chain(std::iter::once(&path)) + .cloned() + .collect::>() + .join(" -> "); + return Err(vec![problem( + bundle, + &at.path, + at.span, + "include_cycle", + format!("Include cycle: {chain}."), + )]); + } + if depth > limits.max_include_depth { + return Err(vec![problem( + bundle, + &at.path, + at.span, + "project_limit", + format!("Include depth exceeds {}.", limits.max_include_depth), + )]); + } + if !visited.insert(path.clone()) { + continue; + } + let Some(source) = bundle.files.get(&path) else { + return Err(vec![problem( + bundle, + &at.path, + at.span, + "missing_include", + format!("Included file '{path}' is missing."), + )]); + }; + let mut unit = language::parse_unit(source, path == bundle.entry, limits.model).map_err( + |diagnostic| { + vec![SourceDiagnostic { + path: path.clone(), + diagnostic: *diagnostic, + related: Vec::new(), + }] + }, + )?; + if let Some(header) = unit.header.take() { + let location = SourceLocation { + path: path.clone(), + span: header.span, + }; + aliases.insert(header.project.alias.clone(), header.project.id); + alias_locations.insert(header.project.alias.clone(), location.clone()); + declaration_locations.insert(header.project.id, location); + if let Some(offset) = header.id_insertion { + insert_id(&mut insertions, &path, source, offset, header.project.id); + } + project_header = Some(header); + } + for declaration in unit.declarations { + let location = SourceLocation { + path: path.clone(), + span: declaration.span, + }; + let duplicate = alias_locations + .get(&declaration.alias) + .map(|previous| ("alias", declaration.alias.clone(), previous)) + .or_else(|| { + declaration_locations + .get(&declaration.id) + .map(|previous| ("identity", declaration.id.to_string(), previous)) + }); + if let Some((kind, value, previous)) = duplicate { + let mut issue = problem( + bundle, + &path, + declaration.span, + "duplicate_declaration", + format!("Duplicate {kind} '{value}'."), + ); + issue.related.push(previous.clone()); + return Err(vec![issue]); + } + aliases.insert(declaration.alias.clone(), declaration.id); + alias_locations.insert(declaration.alias.clone(), location.clone()); + declaration_locations.insert(declaration.id, location); + for (key, span) in &declaration.field_spans { + fields.insert( + (declaration.id, key.clone()), + SourceLocation { + path: path.clone(), + span: *span, + }, + ); + } + if let Some(offset) = declaration.id_insertion { + insert_id(&mut insertions, &path, source, offset, declaration.id); + } + graph_count += usize::from(declaration.is_graph_object()); + limits + .model + .check_counts(graph_count, declarations.len().saturating_add(2), 0) + .map_err(|message| { + vec![problem( + bundle, + &path, + declaration.span, + "model_limit", + message, + )] + })?; + declarations.push(declaration); + declaration_paths.push(path.clone()); + } + let mut children = Vec::new(); + for (raw, span) in unit.includes { + let target = resolve_include(&path, &raw) + .map_err(|message| vec![problem(bundle, &path, span, "invalid_path", message)])?; + let location = SourceLocation { + path: path.clone(), + span, + }; + includes.push(IncludeLocation { + location: location.clone(), + target: target.clone(), + }); + if includes.len() > limits.max_include_edges { + return Err(vec![problem( + bundle, + &path, + span, + "project_limit", + format!("Include graph exceeds {} edges.", limits.max_include_edges), + )]); + } + children.push(Visit::Enter( + target, + depth.saturating_add(1), + Some(location), + )); + } + active.push(path); + stack.push(Visit::Leave); + stack.extend(children.into_iter().rev()); + } + if let Some(path) = bundle.files.keys().find(|path| !visited.contains(*path)) { + return Err(vec![problem( + bundle, + path, + Span::default(), + "unreachable_file", + "Supplied source file is not reachable from the entry file.", + )]); + } + let header = project_header.expect("validated entry header"); + let index = SourceIndex { + declarations: declaration_locations, + fields, + includes, + project_header: SourceLocation { + path: bundle.entry.clone(), + span: header.span, + }, + project_body: SourceLocation { + path: bundle.entry.clone(), + span: header.body, + }, + project_description: header.description.map(|span| SourceLocation { + path: bundle.entry.clone(), + span, + }), + }; + let project = + language::resolve_declarations(header.project, &declarations, &aliases, limits.model) + .map_err(|(index, error)| { + let path = declaration_paths.get(index).unwrap_or(&bundle.entry); + vec![SourceDiagnostic { + path: path.clone(), + diagnostic: language::diagnostic(&bundle.files[path], error), + related: Vec::new(), + }] + })?; + let diagnostics = model_diagnostics(bundle, &index, &project); + let errors: Vec<_> = diagnostics + .iter() + .filter(|issue| issue.diagnostic.severity == validation::Severity::Error) + .cloned() + .collect(); + if !errors.is_empty() { + return Err(errors); + } + Ok(( + CompiledSourceProject { + bundle: bundle.clone(), + decoded: DecodeResult { + project, + migration: None, + }, + index, + diagnostics, + }, + insertions, + )) +} + +fn model_diagnostics( + bundle: &SourceBundle, + index: &SourceIndex, + project: &Project, +) -> Vec { + let line_starts: BTreeMap<_, Vec<_>> = bundle + .files + .iter() + .map(|(path, source)| { + ( + path, + std::iter::once(0) + .chain(source.match_indices('\n').map(|(offset, _)| offset + 1)) + .collect(), + ) + }) + .collect(); + validation::validate(project) + .into_iter() + .map(|issue| { + let location = issue + .element + .and_then(|id| { + issue + .field + .as_ref() + .and_then(|field| index.fields.get(&(id, field.clone()))) + .or_else(|| index.declarations.get(&id)) + }) + .unwrap_or(&index.project_header); + let source = &bundle.files[&location.path]; + let starts = &line_starts[&location.path]; + let line = starts.partition_point(|start| *start <= location.span.start); + SourceDiagnostic { + path: location.path.clone(), + related: Vec::new(), + diagnostic: Diagnostic { + severity: issue.severity, + code: issue.code, + object_id: issue.element, + field: issue.field, + message: issue.message, + span: location.span, + line, + column: source[starts[line - 1]..location.span.start] + .chars() + .count() + + 1, + }, + } + }) + .collect() +} + +fn insert_id(edits: &mut Edits, path: &str, source: &str, offset: usize, id: Id) { + let space = if source[..offset] + .chars() + .next_back() + .is_some_and(char::is_whitespace) + { + "" + } else { + " " + }; + edits.entry(path.into()).or_default().push(( + Span { + start: offset, + end: offset, + }, + format!("{space}id \"{id}\" "), + )); +} + +fn apply_edits(bundle: &mut SourceBundle, edits: Edits) -> Result<(), Vec> { + for (path, changes) in edits { + let source = &bundle.files[&path]; + let mut grouped: BTreeMap<(usize, usize), String> = BTreeMap::new(); + for (span, text) in changes { + if span.start > span.end + || span.end > source.len() + || !source.is_char_boundary(span.start) + || !source.is_char_boundary(span.end) + { + return Err(vec![problem( + bundle, + &path, + Span::default(), + "stale_source_index", + "Source edit positions no longer match the document.", + )]); + } + grouped + .entry((span.start, span.end)) + .or_default() + .push_str(&text); + } + let mut end = 0; + for &(start, next_end) in grouped.keys() { + if start < end { + return Err(vec![problem( + bundle, + &path, + Span::default(), + "stale_source_index", + "Source edits overlap.", + )]); + } + end = next_end; + } + let source = bundle.files.get_mut(&path).expect("indexed file"); + for ((start, end), text) in grouped.into_iter().rev() { + source.replace_range(start..end, &text); + } + } + Ok(()) +} + +/// Canonical, readable source3 export. Like `language::format`, the caller is +/// responsible for supplying a valid model; compilation checks the result. +/// Existing source layout is preserved by `update_bundle`, not this formatter. +pub fn format_bundle(project: &Project) -> SourceBundle { + let ownership = default_ownership(project); + let aliases: BTreeMap<_, _> = project.identities().into_iter().collect(); + let mut declarations = language::format_declarations(project); + declarations.sort_by_key(|(id, _)| declaration_order(project, *id, &aliases)); + let mut files: BTreeMap = BTreeMap::new(); + for (id, declaration) in declarations { + files + .entry(ownership[&id].clone()) + .or_default() + .push_str(&unindent(&declaration)); + } + let mut entry = format!( + "arcadia 3;\n{}{{\n description {};\n", + project_header(project), + quote(&project.description) + ); + for path in files.keys() { + entry.push_str(&format!(" include {};\n", quote(path))); + } + entry.push_str("}\n"); + files.insert("project.arcadia".into(), entry); + SourceBundle { + entry: "project.arcadia".into(), + files, + } +} + +/// Apply a model change to a compiled document workspace. Existing declaration +/// ownership and untouched bytes remain stable; only changed declarations are +/// canonicalized. Alias edits also rewrite declarations that reference them. +/// No files are deleted, even when their last declaration is removed. +pub fn update_bundle( + current: &CompiledSourceProject, + project: &Project, + limits: ProjectLimits, +) -> Result> { + // Public document/index fields are convenient for callers, but cannot become + // an unchecked editing authority if a caller has changed either of them. + let checked = compile_bundle(¤t.bundle, limits)?; + if checked.index != current.index + || checked.decoded.project != current.decoded.project + || checked.bundle != current.bundle + { + return Err(vec![problem( + ¤t.bundle, + ¤t.bundle.entry, + Span::default(), + "stale_source_index", + "Compile the current documents before applying a model update.", + )]); + } + let errors: Vec<_> = model_diagnostics(¤t.bundle, ¤t.index, project) + .into_iter() + .filter(|issue| issue.diagnostic.severity == validation::Severity::Error) + .collect(); + if !errors.is_empty() { + return Err(errors); + } + limits.model.check_model(project).map_err(|message| { + vec![problem( + ¤t.bundle, + ¤t.bundle.entry, + Span::default(), + "model_limit", + message, + )] + })?; + let old = ¤t.decoded.project; + let before: BTreeMap<_, _> = language::format_declarations(old).into_iter().collect(); + let after: BTreeMap<_, _> = language::format_declarations(project).into_iter().collect(); + let mut bundle = current.bundle.clone(); + let mut edits: Edits = BTreeMap::new(); + for (id, old_text) in &before { + let location = ¤t.index.declarations[id]; + if after.get(id) != Some(old_text) { + let text = after.get(id).map_or_else(String::new, |text| { + replacement(text, &bundle.files[&location.path], location.span.start) + }); + edits + .entry(location.path.clone()) + .or_default() + .push((location.span, text)); + } + } + if old.id != project.id || old.alias != project.alias || old.name != project.name { + let location = ¤t.index.project_header; + edits + .entry(location.path.clone()) + .or_default() + .push((location.span, project_header(project))); + } + if old.description != project.description { + let field = format!("description {};", quote(&project.description)); + if let Some(location) = ¤t.index.project_description { + edits + .entry(location.path.clone()) + .or_default() + .push((location.span, field)); + } else { + let location = ¤t.index.project_body; + edits.entry(location.path.clone()).or_default().push(( + Span { + start: location.span.start, + end: location.span.start, + }, + format!("\n {field}\n"), + )); + } + } + + let defaults = default_ownership(project); + let mut assigned: BTreeMap<_, _> = current + .index + .declarations + .iter() + .filter(|(id, _)| project.contains(**id)) + .map(|(id, location)| (*id, location.path.clone())) + .collect(); + let mut chosen_defaults = BTreeMap::new(); + let mut used_paths: BTreeSet<_> = bundle + .files + .keys() + .map(|path| path.to_lowercase()) + .collect(); + let mut additions: BTreeMap = BTreeMap::new(); + let aliases: BTreeMap<_, _> = project.identities().into_iter().collect(); + let mut new_ids: Vec<_> = after + .keys() + .filter(|id| !before.contains_key(*id)) + .copied() + .collect(); + new_ids.sort_by_key(|id| declaration_order(project, *id, &aliases)); + for id in new_ids { + let mut parent = semantic_owner(project, id); + let mut seen = BTreeSet::new(); + let mut inherited = None; + while let Some(owner) = parent { + if !seen.insert(owner) { + break; + } + if let Some(path) = assigned.get(&owner) { + inherited = Some(path.clone()); + break; + } + parent = semantic_owner(project, owner); + } + let path = inherited.unwrap_or_else(|| { + chosen_defaults + .entry(defaults[&id].clone()) + .or_insert_with(|| { + let preferred = &defaults[&id]; + if bundle.files.contains_key(preferred) { + preferred.clone() + } else { + unique_path(preferred, &mut used_paths) + } + }) + .clone() + }); + assigned.insert(id, path.clone()); + additions + .entry(path) + .or_default() + .push_str(&unindent(&after[&id])); + } + let mut new_includes = String::new(); + for (path, text) in additions { + if !bundle.files.contains_key(&path) { + bundle.files.insert(path.clone(), String::new()); + new_includes.push_str(&format!(" include {};\n", quote(&path))); + } + let at = if path == bundle.entry { + current.index.project_body.span.end + } else { + bundle.files[&path].len() + }; + edits + .entry(path) + .or_default() + .push((Span { start: at, end: at }, format!("\n{text}"))); + } + if !new_includes.is_empty() { + let at = current.index.project_body.span.end; + edits + .entry(bundle.entry.clone()) + .or_default() + .push((Span { start: at, end: at }, format!("\n{new_includes}"))); + } + apply_edits(&mut bundle, edits)?; + let result = compile_bundle(&bundle, limits)?; + if &result.decoded.project != project { + return Err(vec![problem( + &bundle, + &bundle.entry, + Span::default(), + "source_update_mismatch", + "Updated documents do not reproduce the requested model.", + )]); + } + Ok(result) +} + +fn quote(text: &str) -> String { + serde_json::to_string(text).expect("string serialization") +} +fn project_header(project: &Project) -> String { + format!( + "project {} {} id \"{}\" ", + project.alias, + quote(&project.name), + project.id + ) +} +fn unindent(text: &str) -> String { + text.split_inclusive('\n') + .map(|line| line.strip_prefix(" ").unwrap_or(line)) + .collect() +} +fn replacement(text: &str, source: &str, start: usize) -> String { + let prefix = source[..start].rsplit('\n').next().unwrap_or(""); + let indent = if prefix.chars().all(char::is_whitespace) { + prefix + } else { + "" + }; + let text = unindent(text); + let text = text.trim_end().replace('\n', &format!("\n{indent}")); + if source.contains("\r\n") { + text.replace('\n', "\r\n") + } else { + text + } +} + +fn semantic_owner(project: &Project, id: Id) -> Option { + if let Some(element) = project.elements.get(&id) { + return element.parent; + } + if let Some(relation) = project.relations.get(&id) { + return Some(relation.source()); + } + if let Some(artifact) = project.artifact_links.get(&id) { + return Some(artifact.owner); + } + project + .diagrams + .values() + .find(|diagram| diagram.nodes.contains_key(&id)) + .map(|diagram| diagram.id) +} + +fn default_module(project: &Project, id: Id) -> LogicalPath { + if let Some(diagram) = project.diagrams.get(&id) { + return format!("views/diagram_{}.arcadia", short_alias(&diagram.alias)); + } + if let Some(diagram) = project + .diagrams + .values() + .find(|diagram| diagram.nodes.contains_key(&id)) + { + return format!("views/diagram_{}.arcadia", short_alias(&diagram.alias)); + } + if project.traces.contains_key(&id) { + return "traceability.arcadia".into(); + } + let mut ancestors = Vec::new(); + let mut current = Some(id); + let mut seen = BTreeSet::new(); + while let Some(id) = current { + if !seen.insert(id) { + break; + } + if let Some(element) = project.elements.get(&id) { + ancestors.push(element); + } + current = semantic_owner(project, id); + } + let Some(element) = ancestors.first() else { + return "artifacts.arcadia".into(); + }; + let layer = element.layer().to_string().to_lowercase(); + if let Some(context) = ancestors.iter().find(|e| { + matches!( + e.kind(), + Kind::Scenario | Kind::StateMachine | Kind::Process | Kind::FunctionalChain + ) + }) { + let kind = match context.kind() { + Kind::Scenario => "scenario", + Kind::StateMachine => "state_machine", + _ => "process", + }; + return format!( + "{layer}/behavior/{kind}_{}.arcadia", + short_alias(&context.alias) + ); + } + if let Some(work) = ancestors + .iter() + .rev() + .find(|e| matches!(e.kind(), Kind::Activity | Kind::Function)) + { + let (folder, prefix) = if work.kind() == Kind::Activity { + ("activities", "activity") + } else { + ("functions", "function") + }; + return format!( + "{layer}/{folder}/{prefix}_{}.arcadia", + short_alias(&work.alias) + ); + } + let root = ancestors.last().expect("nonempty ancestors"); + let category = match root.kind() { + Kind::Entity | Kind::Actor | Kind::Role | Kind::System | Kind::SystemActor => { + "participants" + } + Kind::DataType | Kind::DataField | Kind::ExchangeItem | Kind::Interface => "types", + Kind::Capability | Kind::SystemCapability | Kind::Mission | Kind::OperationalMission => { + "capabilities" + } + _ => "model", + }; + format!("{layer}/{category}.arcadia") +} + +fn short_alias(alias: &str) -> String { + alias.chars().take(64).collect() +} +fn unique_path(preferred: &str, used: &mut BTreeSet) -> LogicalPath { + let mut result = preferred.to_owned(); + let mut number = 2usize; + while !used.insert(result.to_lowercase()) { + result = format!( + "{}_{number}.arcadia", + preferred.trim_end_matches(".arcadia") + ); + number += 1; + } + result +} +fn default_ownership(project: &Project) -> BTreeMap { + let mut ownership: BTreeMap<_, _> = project + .identities() + .into_iter() + .filter(|(id, _)| *id != project.id) + .map(|(id, _)| (id, default_module(project, id))) + .collect(); + let distinct: BTreeSet<_> = ownership.values().cloned().collect(); + // Large sets of independent roots remain readable category modules instead + // of producing thousands of tiny files or violating the default file cap. + if distinct.len() > 240 { + for path in ownership.values_mut() { + if let Some((folder, _)) = path.rsplit_once('/') { + *path = format!("{folder}/module.arcadia"); + } + } + } + let mut used = BTreeSet::from(["project.arcadia".into()]); + let paths: BTreeSet<_> = ownership.values().cloned().collect(); + let remap: BTreeMap<_, _> = paths + .into_iter() + .map(|path| { + let chosen = unique_path(&path, &mut used); + (path, chosen) + }) + .collect(); + for path in ownership.values_mut() { + *path = remap[path].clone(); + } + ownership +} + +fn declaration_order(project: &Project, id: Id, aliases: &BTreeMap) -> (u8, Vec) { + let rank = if project.elements.contains_key(&id) || project.diagrams.contains_key(&id) { + 0 + } else if project.relations.contains_key(&id) { + 2 + } else if project.artifact_links.contains_key(&id) { + 3 + } else if project.traces.contains_key(&id) { + 4 + } else { + 1 + }; + let mut names = Vec::new(); + let mut seen = BTreeSet::new(); + let mut current = Some(id); + while let Some(id) = current { + if !seen.insert(id) { + break; + } + names.push(aliases.get(&id).copied().unwrap_or("").to_owned()); + current = semantic_owner(project, id); + } + names.reverse(); + (rank, names) +} diff --git a/crates/oa-core/tests/source_project.rs b/crates/oa-core/tests/source_project.rs new file mode 100644 index 00000000..f280ecc1 --- /dev/null +++ b/crates/oa-core/tests/source_project.rs @@ -0,0 +1,688 @@ +use oa_core::{language, source_project::*, *}; +use std::collections::BTreeMap; + +fn bundle(entry: &str, fragments: &[(&str, &str)]) -> SourceBundle { + let mut files = BTreeMap::from([("project.arcadia".into(), entry.into())]); + files.extend( + fragments + .iter() + .map(|(path, text)| ((*path).into(), (*text).into())), + ); + SourceBundle { + entry: "project.arcadia".into(), + files, + } +} + +fn compile(bundle: &SourceBundle) -> CompiledSourceProject { + compile_bundle(bundle, ProjectLimits::default()).unwrap() +} + +fn id(project: &Project, alias: &str) -> Id { + project + .identities() + .into_iter() + .find(|(_, candidate)| *candidate == alias) + .unwrap() + .0 +} + +fn typed() -> SourceBundle { + bundle( + "arcadia 3; project test_model \"航海\" {\n include \"exchanges.arcadia\";\n include \"work/send.arcadia\";\n include \"work/receive.arcadia\";\n include \"types.arcadia\";\n}", + &[ + ( + "exchanges.arcadia", + "// Boundary-independent functional exchange\nfunctional_exchange message \"Message\" { source @send; target @receive; source_port @output; target_port @input; items [@data]; }\n", + ), + ( + "work/send.arcadia", + "// keep this explanation\nsa_function send \"Send\" {}\nsa_port output \"Output\" { parent @send; contract { FunctionOutput { outgoing_items [@data]; }; }; }\n", + ), + ( + "work/receive.arcadia", + "sa_function receive \"Receive\" {}\nsa_port input \"Input\" { parent @receive; contract { FunctionInput { incoming_items [@data]; }; }; }\n", + ), + ("types.arcadia", "sa_exchange_item data \"Data\" {}\n"), + ], + ) +} + +#[test] +fn cross_file_typed_ports_global_forward_references_and_stable_ids() { + let input = typed(); + let original = input.clone(); + let result = compile(&input); + assert_eq!(input, original, "compilation must not mutate caller drafts"); + assert_ne!(result.bundle, input); + let again = compile(&result.bundle); + assert_eq!(again.decoded.project, result.decoded.project); + assert_eq!(again.bundle, result.bundle); + assert_eq!(again.index, result.index); + assert!(result.decoded.migration.is_none()); + let project = &result.decoded.project; + assert_eq!( + project.relations[&id(project, "message")].source(), + id(project, "send") + ); + let port = id(project, "output"); + assert_eq!(result.index.declarations[&port].path, "work/send.arcadia"); + let field = &result.index.fields[&(port, "contract".into())]; + assert_eq!( + &result.bundle.files[&field.path][field.span.start..field.span.end], + "{ FunctionOutput { outgoing_items [@data]; }; }" + ); + assert_eq!( + language::parse(&language::format(project)).unwrap(), + *project + ); + assert_eq!( + persistence::decode(&serde_json::to_vec(project).unwrap()).unwrap(), + *project + ); + assert!(result.bundle.files["work/send.arcadia"].starts_with("// keep this explanation\n")); + let send = id(project, "send"); + let warning = result + .diagnostics + .iter() + .find(|issue| { + issue.diagnostic.object_id == Some(send) && issue.diagnostic.code == "unallocated_leaf" + }) + .unwrap(); + assert_eq!(warning.path, "work/send.arcadia"); + assert_eq!(warning.diagnostic.line, 2); +} + +#[test] +fn source_errors_keep_exact_fragment_unicode_positions() { + let input = bundle( + "arcadia 3; project p \"P\" { include \"child.arcadia\"; }", + &[( + "child.arcadia", + "// Пример\nsa_function child \"航海\" { parent @missing; }", + )], + ); + let error = compile_bundle(&input, ProjectLimits::default()) + .unwrap_err() + .remove(0); + let text = &input.files["child.arcadia"]; + assert_eq!(error.path, "child.arcadia"); + assert_eq!(error.diagnostic.line, 2); + let offset = text.find("@missing").unwrap(); + assert_eq!(error.diagnostic.span.start, offset); + assert_eq!( + error.diagnostic.column, + text[..offset].rsplit('\n').next().unwrap().chars().count() + 1 + ); + assert!(error.diagnostic.message.contains("@missing")); + let cyclic_parent = SourceBundle { + files: input + .files + .iter() + .map(|(path, text)| (path.clone(), text.replace("@missing", "@child"))) + .collect(), + ..input + }; + let error = compile_bundle(&cyclic_parent, ProjectLimits::default()).unwrap_err(); + let parent = error + .iter() + .find(|issue| issue.diagnostic.code == "invalid_parent") + .unwrap(); + assert_eq!(parent.path, "child.arcadia"); + assert_eq!(parent.diagnostic.span.start, offset); +} + +#[test] +fn duplicate_alias_and_identity_report_both_files_including_project_identity() { + for (left, right) in [ + ("oa_activity same \"A\" {}", "oa_activity same \"B\" {}"), + ( + "oa_activity first \"A\" id \"00000000-0000-4000-8000-000000000001\" {}", + "oa_activity second \"B\" id \"00000000-0000-4000-8000-000000000001\" {}", + ), + ] { + let input = bundle( + "arcadia 3; project p \"P\" { include \"a.arcadia\"; include \"b.arcadia\"; }", + &[("a.arcadia", left), ("b.arcadia", right)], + ); + let error = compile_bundle(&input, ProjectLimits::default()) + .unwrap_err() + .remove(0); + assert_eq!(error.diagnostic.code, "duplicate_declaration"); + assert_eq!(error.path, "b.arcadia"); + assert_eq!(error.related[0].path, "a.arcadia"); + } + let input = bundle( + "arcadia 3; project p \"P\" { include \"child.arcadia\"; }", + &[("child.arcadia", "oa_activity p \"Collision\" {}")], + ); + let error = compile_bundle(&input, ProjectLimits::default()) + .unwrap_err() + .remove(0); + assert_eq!(error.related[0].path, "project.arcadia"); +} + +#[test] +fn includes_are_once_only_and_cycles_missing_and_unreachable_files_are_located() { + let mut input = bundle( + "arcadia 3; project p \"P\" { include \"sub/a.arcadia\"; include \"sub/b.arcadia\"; }", + &[ + ("sub/a.arcadia", "include \"../shared.arcadia\";"), + ("sub/b.arcadia", "include \"../shared.arcadia\";"), + ("shared.arcadia", "oa_activity shared_work \"Shared\" {}"), + ], + ); + assert_eq!(compile(&input).decoded.project.elements.len(), 1); + input + .files + .insert("shared.arcadia".into(), "include \"sub/a.arcadia\";".into()); + let error = compile_bundle(&input, ProjectLimits::default()) + .unwrap_err() + .remove(0); + assert_eq!(error.path, "shared.arcadia"); + assert_eq!(error.diagnostic.code, "include_cycle"); + assert!( + error + .diagnostic + .message + .contains("sub/a.arcadia -> shared.arcadia -> sub/a.arcadia") + ); + input.files.remove("shared.arcadia"); + let error = compile_bundle(&input, ProjectLimits::default()) + .unwrap_err() + .remove(0); + assert_eq!(error.path, "sub/a.arcadia"); + assert_eq!(error.diagnostic.code, "missing_include"); + input.files.insert("shared.arcadia".into(), String::new()); + input.files.insert("unused.arcadia".into(), String::new()); + assert_eq!( + compile_bundle(&input, ProjectLimits::default()).unwrap_err()[0] + .diagnostic + .code, + "unreachable_file" + ); +} + +#[test] +fn portable_paths_reject_escapes_devices_urls_and_case_collisions() { + for path in [ + "../escape.arcadia", + "/abs.arcadia", + "C:/drive.arcadia", + "https://host/a.arcadia", + "a\\b.arcadia", + "a//b.arcadia", + "con.arcadia", + "dir./a.arcadia", + "a.txt", + ] { + assert!(resolve_include("project.arcadia", path).is_err(), "{path}"); + } + assert_eq!( + resolve_include("sub/a.arcadia", "../b.arcadia").unwrap(), + "b.arcadia" + ); + assert!(resolve_include("../bad.arcadia", "a.arcadia").is_err()); + let input = bundle( + "arcadia 3; project p \"P\" {}", + &[("A.arcadia", ""), ("a.arcadia", "")], + ); + assert_eq!( + compile_bundle(&input, ProjectLimits::default()).unwrap_err()[0] + .diagnostic + .code, + "duplicate_path" + ); +} + +#[test] +fn project_admission_is_aggregate_and_precedes_global_resolution() { + let input = bundle( + "arcadia 3; project p \"P\" { include \"a.arcadia\"; include \"b.arcadia\"; }", + &[ + ("a.arcadia", "oa_activity one \"One\" { parent @missing; }"), + ("b.arcadia", "oa_activity two \"Two\" {}"), + ], + ); + let cases = [ + ProjectLimits { + max_files: 2, + ..ProjectLimits::default() + }, + ProjectLimits { + max_include_depth: 0, + ..ProjectLimits::default() + }, + ProjectLimits { + max_include_edges: 1, + ..ProjectLimits::default() + }, + ProjectLimits { + model: language::ParseLimits { + max_graph_objects: 1, + ..language::ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }, + ProjectLimits { + model: language::ParseLimits { + max_records: 2, + ..language::ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }, + ProjectLimits { + model: language::ParseLimits { + max_source_bytes: input.files.values().map(String::len).sum::() - 1, + ..language::ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + }, + ]; + for limits in cases { + let error = compile_bundle(&input, limits).unwrap_err(); + assert!(error[0].diagnostic.code.ends_with("limit"), "{error:?}"); + assert!(!error[0].diagnostic.message.contains("@missing")); + } +} + +#[test] +fn canonical_export_groups_semantic_roots_ports_and_placements_and_is_deterministic() { + let mut project = compile(&typed()).decoded.project; + let child = project + .add_element( + Kind::Function, + Layer::Sa, + "Child", + Some(id(&project, "send")), + ) + .unwrap(); + let diagram = project.ensure_diagram(Layer::Sa); + let node = project.place(diagram, child, Position { x: 40., y: 80. }); + let output = format_bundle(&project); + assert_eq!(format_bundle(&project), output); + let result = compile(&output); + assert_eq!(result.decoded.project, project); + assert!( + result.index.declarations[&child] + .path + .contains("function_send.arcadia") + ); + assert_eq!( + result.index.declarations[&child].path, + result.index.declarations[&id(&project, "output")].path + ); + assert_eq!( + result.index.declarations[&node].path, + result.index.declarations[&diagram].path + ); + let work = &output.files[&result.index.declarations[&child].path]; + assert!( + work.find("sa_function send ").unwrap() + < work + .find(&format!("sa_function {} ", project.elements[&child].alias)) + .unwrap() + ); +} + +#[test] +fn moving_declarations_between_files_changes_no_semantics_or_ids() { + let compiled = compile(&typed()); + let mut moved = compiled.bundle.clone(); + let port = id(&compiled.decoded.project, "output"); + let location = &compiled.index.declarations[&port]; + let declaration = + moved.files[&location.path][location.span.start..location.span.end].to_owned(); + moved + .files + .get_mut(&location.path) + .unwrap() + .replace_range(location.span.start..location.span.end, ""); + moved + .files + .get_mut("exchanges.arcadia") + .unwrap() + .push_str(&format!("\n{declaration}\n")); + let moved = compile(&moved); + assert_eq!(moved.decoded.project, compiled.decoded.project); + assert_eq!(moved.index.declarations[&port].path, "exchanges.arcadia"); +} + +#[test] +fn model_updates_rewrite_cross_file_alias_uses_preserve_ownership_and_untouched_comments() { + let compiled = compile(&typed()); + let mut project = compiled.decoded.project.clone(); + let send = id(&project, "send"); + project.elements.get_mut(&send).unwrap().alias = "transmit".into(); + project.elements.get_mut(&send).unwrap().name = "Transmit data".into(); + project.description = "Documented project".into(); + let result = update_bundle(&compiled, &project, ProjectLimits::default()).unwrap(); + assert_eq!(result.decoded.project, project); + assert_eq!(result.index.declarations[&send].path, "work/send.arcadia"); + assert!(result.bundle.files["exchanges.arcadia"].contains("source @transmit;")); + assert!(result.bundle.files["work/send.arcadia"].contains("parent @transmit;")); + assert!(result.bundle.files["work/send.arcadia"].starts_with("// keep this explanation\n")); + assert_eq!( + result.bundle.files["types.arcadia"], + compiled.bundle.files["types.arcadia"] + ); + assert_eq!( + result.bundle.files["work/receive.arcadia"], + compiled.bundle.files["work/receive.arcadia"] + ); + assert_eq!( + update_bundle(&result, &project, ProjectLimits::default()) + .unwrap() + .bundle, + result.bundle + ); +} + +#[test] +fn graphical_move_updates_only_its_owned_view_document() { + let mut project = compile(&typed()).decoded.project; + let diagram = project.ensure_diagram(Layer::Sa); + let send = id(&project, "send"); + let node = project.place(diagram, send, Position { x: 0., y: 0. }); + let current = compile(&format_bundle(&project)); + project + .diagrams + .get_mut(&diagram) + .unwrap() + .nodes + .get_mut(&node) + .unwrap() + .position = Position { x: 142., y: 83. }; + let result = update_bundle(¤t, &project, ProjectLimits::default()).unwrap(); + let changed: Vec<_> = result + .bundle + .files + .iter() + .filter(|(path, text)| current.bundle.files[*path] != **text) + .map(|(path, _)| path.clone()) + .collect(); + assert_eq!( + changed, + vec![current.index.declarations[&node].path.clone()] + ); + assert_eq!(result.decoded.project, project); +} + +#[test] +fn new_children_inherit_owner_file_new_roots_add_include_and_deletion_keeps_files() { + let current = compile(&typed()); + let mut project = current.decoded.project.clone(); + let send = id(&project, "send"); + let child = project + .add_element(Kind::Function, Layer::Sa, "Nested work", Some(send)) + .unwrap(); + let root = project + .add_element(Kind::Function, Layer::Sa, "Independent work", None) + .unwrap(); + let result = update_bundle(¤t, &project, ProjectLimits::default()).unwrap(); + assert_eq!(result.index.declarations[&child].path, "work/send.arcadia"); + let root_file = result.index.declarations[&root].path.clone(); + assert!(root_file.contains("independent_work")); + assert!(result.bundle.files["project.arcadia"].contains(&format!("include \"{root_file}\";"))); + project.elements.remove(&root); + let deleted = update_bundle(&result, &project, ProjectLimits::default()).unwrap(); + assert!(deleted.bundle.files.contains_key(&root_file)); + assert!(!deleted.index.declarations.contains_key(&root)); +} + +#[test] +fn failed_update_preserves_candidate_and_rejects_stale_index() { + let current = compile(&typed()); + let original = current.bundle.clone(); + let mut project = current.decoded.project.clone(); + project + .elements + .get_mut(&id(¤t.decoded.project, "send")) + .unwrap() + .parent = Some(Id::new_v4()); + assert!(update_bundle(¤t, &project, ProjectLimits::default()).is_err()); + assert_eq!(current.bundle, original); + let mut stale = current.clone(); + stale + .bundle + .files + .get_mut("work/send.arcadia") + .unwrap() + .insert_str(0, "// external edit\n"); + assert_eq!( + update_bundle(&stale, ¤t.decoded.project, ProjectLimits::default()).unwrap_err()[0] + .diagnostic + .code, + "stale_source_index" + ); +} + +#[test] +fn single_string_and_legacy_apis_remain_explicit_and_filesystem_free() { + let source = "arcadia 3; project p \"P\" { include \"missing.arcadia\"; }"; + assert!( + language::parse(source).unwrap_err()[0] + .message + .contains("requires a source bundle") + ); + let old = "arcadia 2; project p \"Legacy\" { oa_activity work \"Work\" {} }"; + let decoded = language::parse_with_report(old).unwrap(); + assert!(decoded.migration.is_some()); + assert!(compile_bundle(&bundle(old, &[]), ProjectLimits::default()).is_err()); + assert_eq!( + compile(&format_bundle(&decoded.project)).decoded.project, + decoded.project + ); +} + +#[test] +fn exported_case_colliding_root_aliases_are_portable_and_large_root_sets_are_bounded() { + let mut project = Project::empty(); + let a = project + .add_element(Kind::Activity, Layer::Oa, "Upper", None) + .unwrap(); + let b = project + .add_element(Kind::Activity, Layer::Oa, "Lower", None) + .unwrap(); + project.elements.get_mut(&a).unwrap().alias = "Work".into(); + project.elements.get_mut(&b).unwrap().alias = "work".into(); + let result = compile(&format_bundle(&project)); + assert_ne!( + result.index.declarations[&a].path.to_lowercase(), + result.index.declarations[&b].path.to_lowercase() + ); + for n in 0..260 { + project + .add_element(Kind::Activity, Layer::Oa, &format!("Activity {n}"), None) + .unwrap(); + } + let generated = format_bundle(&project); + assert!(generated.files.len() <= ProjectLimits::default().max_files); + assert_eq!(compile(&generated).decoded.project, project); +} + +#[test] +fn complete_demo_preserves_behavior_artifacts_traces_and_every_placement() { + let project = Project::demo(); + let generated = format_bundle(&project); + let result = compile(&generated); + assert_eq!(result.decoded.project, project); + for element in project + .elements + .values() + .filter(|e| matches!(e.kind(), Kind::Scenario | Kind::StateMachine)) + { + assert!( + result.index.declarations[&element.id] + .path + .contains("/behavior/") + ); + for child in project + .elements + .values() + .filter(|child| child.parent == Some(element.id)) + { + assert_eq!( + result.index.declarations[&child.id].path, + result.index.declarations[&element.id].path + ); + } + } + let json = serde_json::to_string(&generated).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + generated + ); +} + +#[test] +fn json_bundle_rejects_duplicate_file_keys_and_unknown_fields() { + assert!(serde_json::from_str::(r#"{"entry":"project.arcadia","files":{"project.arcadia":"first","project.arcadia":"second"}}"#).unwrap_err().to_string().contains("Duplicate source path")); + assert!( + serde_json::from_str::( + r#"{"entry":"project.arcadia","files":{},"root":"C:/host"}"# + ) + .is_err() + ); +} + +#[test] +fn generated_id_insertion_handles_adjacent_header_tokens_and_remains_exactly_indexed() { + let input = bundle( + "arcadia 3;project p\"P\"{include \"child.arcadia\";}", + &[( + "child.arcadia", + "oa_activity a\"A\"{}diagram d\"D\"{}node n{diagram @d;element @a;}", + )], + ); + let result = compile(&input); + for (id, location) in &result.index.declarations { + assert!( + result.bundle.files[&location.path][location.span.start..location.span.end] + .contains(&id.to_string()) + ); + } + assert_eq!( + compile(&result.bundle).decoded.project, + result.decoded.project + ); +} + +#[test] +fn duplicate_include_edges_count_toward_admission_and_ordered_references_are_aggregate() { + let input = bundle( + "arcadia 3; project p \"P\" { include \"a.arcadia\"; include \"a.arcadia\"; }", + &[("a.arcadia", "")], + ); + assert_eq!( + compile_bundle( + &input, + ProjectLimits { + max_include_edges: 1, + ..ProjectLimits::default() + } + ) + .unwrap_err()[0] + .diagnostic + .code, + "project_limit" + ); + let input = bundle( + "arcadia 3; project p \"P\" { include \"a.arcadia\"; include \"b.arcadia\"; }", + &[ + ( + "a.arcadia", + "oa_process one \"One\" { steps [@step_one]; } oa_occurrence step_one \"Step one\" { parent @one; body { Control { kind Choice; }; }; }", + ), + ( + "b.arcadia", + "oa_process two \"Two\" { steps [@step_two]; } oa_occurrence step_two \"Step two\" { parent @two; body { Control { kind Choice; }; }; }", + ), + ], + ); + assert_eq!( + compile_bundle( + &input, + ProjectLimits { + model: language::ParseLimits { + max_step_references: 1, + ..language::ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + } + ) + .unwrap_err()[0] + .diagnostic + .code, + "model_limit" + ); +} + +#[test] +fn include_inspection_reuses_strict_entry_fragment_syntax_and_real_spans() { + let source = "// include \"fake.arcadia\";\noa_activity work \"航海\" { parent @external_work; description \"include ignored\"; }\ninclude \"../shared.arcadia\";"; + let includes = + inspect_includes("sub/work.arcadia", source, false, ProjectLimits::default()).unwrap(); + assert_eq!(includes.len(), 1); + assert_eq!(includes[0].target, "shared.arcadia"); + assert_eq!(includes[0].location.path, "sub/work.arcadia"); + let span = includes[0].location.span; + assert_eq!( + &source[span.start..span.end], + "include \"../shared.arcadia\";" + ); + let entry = "arcadia 3; project p \"P\" { include \"sub/work.arcadia\"; }"; + assert_eq!( + inspect_includes("project.arcadia", entry, true, ProjectLimits::default()).unwrap()[0] + .target, + "sub/work.arcadia" + ); + assert!(inspect_includes("project.arcadia", entry, false, ProjectLimits::default()).is_err()); +} + +#[test] +fn include_inspection_returns_located_failure_without_partial_discovery_and_is_bounded() { + let source = "include \"first.arcadia\";\noa_activity invalid \"航海\" { parent @; }"; + let errors = + inspect_includes("broken.arcadia", source, false, ProjectLimits::default()).unwrap_err(); + assert_eq!(errors[0].path, "broken.arcadia"); + assert_eq!(errors[0].diagnostic.line, 2); + assert_eq!( + errors[0].diagnostic.span.start, + source.find("@;").unwrap() + 1 + ); + assert_eq!( + inspect_includes( + "broken.arcadia", + source, + false, + ProjectLimits { + model: language::ParseLimits { + max_source_bytes: 10, + ..language::ParseLimits::UNLIMITED + }, + ..ProjectLimits::default() + } + ) + .unwrap_err()[0] + .diagnostic + .code, + "source_limit" + ); + let source = "include \"first.arcadia\"; include \"second.arcadia\";"; + let errors = inspect_includes( + "project.arcadia", + source, + false, + ProjectLimits { + max_include_edges: 1, + ..ProjectLimits::default() + }, + ) + .unwrap_err(); + assert_eq!( + errors[0].diagnostic.span.start, + source.find("include \"second").unwrap() + ); + assert_eq!(errors[0].diagnostic.code, "project_limit"); +} diff --git a/docs/language.rst b/docs/language.rst index 391bf38d..2c342f36 100644 --- a/docs/language.rst +++ b/docs/language.rst @@ -745,7 +745,7 @@ sets and enum constructors are closed by sections 3–5. .. code-block:: text - document = "arcadia" "2" ";" project + document = "arcadia" "3" ";" project project = "project" alias string identity? "{" project-entry* "}" project-entry = "description" string ";" | declaration declaration = (kind | "diagram" | "artifact" | "trace") @@ -784,6 +784,87 @@ Ordinary string fields require quoted strings and cannot contain an unquoted constructor records. Typed values MUST NOT nest more than 64 levels; malformed deeply nested drafts produce a diagnostic instead of exhausting the parser stack. +6.1. Modular source3 projects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The pure ``source_project`` core API compiles a virtual source bundle, containing +``entry`` and a ``files`` map from logical paths to UTF-8 strings. Its entry is a +root-level ``.arcadia`` file with the source3 project header above. A project entry +MAY also be ``include string ";"``. An included fragment consists only of ordinary +declarations and include statements; it MUST NOT contain another version header, +project wrapper, or project description. Includes MUST occur between declarations, +never inside a typed field value. For example:: + + arcadia 3; + project rescue "Rescue" id "00000000-0000-4000-8000-000000000001" { + description "Operational model"; + include "oa/participants.arcadia"; + include "oa/activities/activity_respond.arcadia"; + include "views/diagram_operational.arcadia"; + } + +An include path is relative to its containing file. ``.`` and ``..`` segments +normalize within the virtual project root; escaping that root is an error. Bundle +keys MUST already be normalized, use forward slashes and the lowercase +``.arcadia`` suffix, and contain no empty path segments. Absolute, drive, UNC, +URL, backslash, control-character and non-portable Windows device paths are +rejected. Paths ending a segment in dot or space are rejected. Keys that collide +after Unicode lowercasing are rejected. References use the exact normalized +spelling; filesystems do not participate in resolution. + +Every reachable file is compiled once, including through repeated or diamond +includes. Include cycles are errors with their chain and including location. +Missing files, duplicate JSON file keys and supplied unreachable files are +errors. The compiler MUST NOT read local files, network resources, environment +variables or the process working directory. A future native loader is responsible +for translating a directory into this explicit bundle and enforcing physical +path ownership; the pure compiler grants no filesystem access. + +``inspect_includes(path, source, is_entry, limits)`` supports native discovery +before all included documents are available. It uses the same strict entry or +fragment syntax parser, applies document byte/record and local include-edge +budgets, and returns normalized targets with exact directive spans. Comments and +strings that mention include syntax are not directives. A syntax failure anywhere +in the document returns located errors and no partial include list; it does not +scan arbitrary token strings after failure. Global references and engineering +validation are deferred to bundle compilation. A native loader MUST retain the +unparsed draft on failure and enforce aggregate bytes/files/depth/edges while +loading; this inspection call does not own or inspect a directory. + +All files share the existing global alias/UUID namespace. Files add no semantic +namespace, implicit parent, allocation or port contract. Forward references work +across files. Duplicate aliases and identities report the offending declaration +and the previous declaration location, including collisions with the project. +Moving a declaration between files MUST preserve the resulting model and UUIDs. + +``ProjectLimits`` adds file, include-depth and include-edge budgets to the existing +``language::ParseLimits``. Model bytes, graph objects, records and ordered +references are counted across the complete bundle, never reset per file. The +default candidate policy is 256 files, depth 32 (entry depth zero), 1,024 include +edges and 64 MiB combined UTF-8 source. Other model counts remain caller policy. +Repeated include edges count toward the edge budget; file contents count once. +The compiler admits input and stabilized output against the same limits. + +``compile_bundle`` returns either located errors and no candidate, or a +``CompiledSourceProject`` with the validated ``DecodeResult``, stabilized bundle, +derived ``SourceIndex`` and located engineering warnings. It inserts explicit +UUID clauses into headers that omitted them without otherwise reformatting the +input. The returned source index and diagnostics address the returned stabilized +bytes. Clients MUST retain that bundle after successful Apply; recompiling the +original buffers with missing identities would generate fresh identities. + +Bundle diagnostics wrap the existing Diagnostic with a logical path and optional +related locations. Declaration spans, top-level field value spans, project +header/body/description spans and resolved include locations are available in +the source index. Columns remain one-based Unicode scalars within the real file; +spans never refer to concatenated or synthetic source. + +Single-string parse/lint APIs remain filesystem-free and reject include +statements with an explicit source-bundle requirement. V1/V2 single-file imports +continue through their existing migration readers and reports. Modular projects +require source3; migration is performed before modular export, not independently +on included fragments. Modular source does not change the semantic JSON schema. + 7. Canonical output and source tooling -------------------------------------- @@ -806,11 +887,46 @@ relations; diagrams and their nodes follow, then artifacts, then traces. Fields use deterministic name order. Output ends with LF. Alias values come from the model, never from regenerating display names. -Comments and input whitespace are lexical trivia. Canonicalizing or applying -source does not preserve them; the editor MUST explain that behavior. +Comments and input whitespace are lexical trivia to the canonical writer. +Canonical export and the current desktop single-document Apply path do not +preserve them; the editor MUST explain that behavior. This existing path does not +define an acceptable lossless authoring policy for a future project editor. +The pure project compiler in section 6.1 preserves them while inserting missing +identities. Project update/export distinctions are described below. Camera position, selection, source cursor, and undo history are not stored model fields. Layout and collapse state inside NodePlacement are model data. +The pure ``format_bundle`` exporter uses the same typed declaration writer as +``format``. It creates a small entry file, layer/category modules, separate +activity/function roots and behavior contexts, and separate diagrams with their +placements. Ports follow their owner; relations follow their source/context. +Module names derive from persistent aliases, not display names or UUID chunks. +Declarations are ordered by semantic ancestry and alias, with owners before +descendants. Case-colliding generated file names receive deterministic numeric +suffixes. When more than 240 distinct modules would be produced, aggregate +folders consolidate into category modules to avoid thousands of tiny files. +For every valid P, ``compile_bundle(format_bundle(P)).decoded.project == P`` +when the caller supplies sufficient admission budgets. + +``update_bundle(compiled, P, limits)`` is a core prototype update facility. It +checks that the compiled document/index/model snapshot still agrees, preserves +existing declaration file ownership and all untouched document bytes, and +recompiles the result before returning it. New children inherit their owner's +file; new roots use canonical module rules and add an explicit entry include. +Removed declarations do not delete source files. Alias changes also update +declarations containing references to that alias. Invalid or stale updates +return errors without mutating the input workspace. + +The current update facility canonicalizes each changed declaration in full. +It therefore DOES NOT preserve inline comments or custom layout inside that +declaration, even for a single scalar edit. This is an explicit prototype +limitation, not final authoring/save acceptance. A lossless project editor needs +minimal header/field/value edits that preserve unrelated tokens and comments; +the available source spans provide the starting point. Canonical modular export +is a distinct, explicit operation and does not promise original comment/layout +preservation. No native multi-file editor, project save transaction or REST +bundle endpoint is claimed by the pure core APIs. + ``parse`` returns a complete structurally validated candidate or error diagnostics and no model. Engineering warnings alone MUST NOT reject a valid candidate. Parsing may stop at the first syntax/conversion error. Diagnostics diff --git a/harness/release-source-project.md b/harness/release-source-project.md new file mode 100644 index 00000000..f9ef68da --- /dev/null +++ b/harness/release-source-project.md @@ -0,0 +1,56 @@ +# Modular source core candidate + +Status: pure source3 subsystem implemented and tested. This is not native project-editor, filesystem-save, REST-bundle, or final human-authoring acceptance. Includes retain the current declaration syntax; a separate usability review is evaluating more concise nested authoring. + +## Delivered + +- `oa_core::source_project`: serde `SourceBundle { entry, files }`, `ProjectLimits`, `CompiledSourceProject`, located diagnostics and derived `SourceIndex`. +- Shared source3 declaration parser/resolver and typed declaration writer in `language.rs`. Flat APIs and V1/V2 migration remain supported. Flat parsing explicitly rejects includes instead of reading a filesystem. +- Global forward references and duplicate alias/UUID detection across files, including project and placement identities. Errors refer to real file offsets; duplicates include the previous location. Engineering warning positions index each file once. +- Portable relative includes, root containment, canonical map keys, case collisions, repeated includes, cycles, missing/unreachable files, duplicate JSON file keys and aggregate admission checks. +- Generated identities are inserted into their original headers. Returned diagnostics/index address the resulting stabilized buffers. Caller drafts are never mutated. +- Canonical modules follow layers, work roots, behavior contexts and diagrams. Ports follow their owners; placements follow diagrams; relations follow their sources. Existing ownership is retained during model updates. Huge sets of independent roots consolidate by category rather than produce thousands of files. +- Prototype model updates preserve untouched files/text, rewrite changed declarations and alias uses, add new owner/root declarations and includes, and retain empty files after deletion. Every update recompiles and checks exact semantic equality before return. + +## Consumer interface + +```rust +use oa_core::source_project::{compile_bundle, format_bundle, update_bundle, ProjectLimits}; + +let compiled = compile_bundle(&draft_bundle, ProjectLimits::default())?; +// Adopt compiled.decoded.project as one Session change only after success. +// Retain compiled.bundle, not the original draft with omitted UUID clauses. +let updated = update_bundle(&compiled, &edited_project, ProjectLimits::default())?; +let canonical_export = format_bundle(&edited_project); +``` + +`ProjectLimits` contains file/depth/edge budgets and the existing `language::ParseLimits` as `model`. Defaults: 256 files, depth 32, 1,024 include edges, 64 MiB combined UTF-8 text; other semantic counts are caller policy. A server should substitute its current aggregate model budgets and retain its encoded request cap. Input and stabilized output must both fit the supplied limits. + +The entry must be a root-level filename, normally `project.arcadia`; fragments inherit source3 and have no project/version wrapper. Paths are relative to the including file, normalized within the virtual root. `resolve_include(from, include)` exposes the same pure path rule. It checks the containing path too. All actual includes must be supplied explicitly in `files`; no host filesystem or network is consulted. + +For native discovery, `inspect_includes(path, source, is_entry, limits)` uses the shared strict parser before referenced files are available. It returns normalized targets and exact directive spans, excluding comments and strings. A syntax failure returns located diagnostics and no partial include list; retain the draft and surface that error rather than scanning arbitrary strings. Cross-file resolution/engineering validation remain in `compile_bundle`. Inspection checks one document's syntax/bytes/records/include edges; the loader must enforce aggregate admission and physical ownership while traversing. + +`SourceIndex.declarations` maps UUID to file and complete declaration span, including the project header under its UUID. `fields` maps `(UUID, top_level_field)` to its complete value span. `includes` holds each directive location and normalized target. Project header, body interior, and optional complete description-field spans support targeted metadata/include edits. Errors use original draft coordinates; successful compilation returns coordinates in stabilized text. + +`CompiledSourceProject.decoded.migration` is `None` for source3 bundles. Legacy import remains `language::parse_with_report` / `persistence::decode_with_report`; retain that report in the importing UI, then explicitly export the migrated model into source3 modules. Do not independently migrate fragments or fabricate a migration report during subsequent source3 compilation. + +## Explicit authoring limitation and next seam + +`update_bundle` currently canonicalizes an entire changed declaration. It preserves comments outside that declaration and all untouched document bytes, but **drops inline comments/custom layout inside a changed declaration**, including for a scalar-only change. This is a tested core candidate, not the adopted final project-save policy. `format_bundle` is an explicit canonical export and intentionally does not reproduce authored trivia. + +The bounded next improvement is a lossless update representation, not a second semantic model: + +1. Extend parser-derived locations to header alias/name/identity tokens, nested record keys/values, list items, and closing-brace insertion points. Existing top-level field value spans already distinguish unchanged fields. +2. Diff old/new typed values using field paths, then replace only changed literal/reference tokens. Preserve field order, spelling/trivia of unaffected fields, and comments between tokens. Adding a field inserts at the owning record; an omitted default needs insertion only when its semantic value changes. +3. For constructor/list structural edits, use keyed record/list changes with explicit trivia retention. Do not silently fall back to replacing the whole record and call it lossless. Keep explicit canonical export separate. +4. Add exact-byte regressions for guard/name/message edits with comments before, within and after the changed record, reordered authored fields, omitted defaults, nested scenario message values, alias uses and CRLF source. Verify model equality and the unchanged bytes in every case. + +## Verification + +- `cargo test -p oa-core`: **150 tests passed**, including 20 new source-project tests. +- `cargo clippy -p oa-core --all-targets -- -D warnings`: passed. +- Owned Rust files formatted with `rustfmt --edition 2024 --config skip_children=true`. + +The source-project tests cover cross-file typed-port/exchange references, Unicode syntax/structural diagnostics, duplicate declarations with related locations, include diamonds/cycles/missing/unreachable files, portable paths, aggregate byte/object/record/step/file/depth/edge budgets, JSON duplicate keys, UUID stabilization and adjacent tokens, declaration relocation, alias propagation, owner inheritance, new module includes, retained empty files, stale/invalid update rejection, canonical case collisions and bounded large-root output. Full demo equality covers behavior, artifacts, trace baselines and placements. Moving a placement changes only its owning view document. + +No native CUA, editor, server, project-directory loader/save journal, port metamodel or Session changes were performed in this subsystem. Those integration milestones remain open. `docs/language.rst` sections 6.1 and 7 distinguish implemented core behavior from these pending authoring/integration requirements; the obsolete source2 grammar production is corrected to source3.