--- a/crates/oa-core/src/language.rs +++ b/crates/oa-core/src/language.rs @@ -141,14 +141,89 @@ pub diagnostics: Vec, pub declarations: Vec, } +/// Check the decoded spelling required by declaration-header UUID clauses. +/// This is the shared 36-byte/hyphen shape rule, not a hexadecimal parser: +/// callers must additionally parse the UUID. Uppercase hex and JSON escapes +/// that decode to this spelling remain accepted. URN/braced/compact forms do not. +pub fn has_uuid_header_shape(decoded: &str) -> bool { + decoded.len() == 36 + && [8, 13, 18, 23] + .into_iter() + .all(|index| decoded.as_bytes()[index] == b'-') +} + +/// Budgets for one original source buffer. Policy is caller-owned; these +/// limits do not change the grammar or perform model admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OutlineLimits { + pub max_source_bytes: usize, + pub max_tokens: usize, + pub max_declarations: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutlineResource { + SourceBytes, + Tokens, + Declarations, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OutlineLimit { + pub resource: OutlineResource, + pub limit: usize, + pub observed_at_least: usize, + /// Original byte offset where admission stopped; never a rewritten offset. + pub offset: usize, +} + +/// Roles discovered by the existing tolerant declaration walk. Keeping the +/// wrapper preserves DeclarationSpan and Analysis as existing public APIs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutlineDeclaration { + pub declaration: DeclarationSpan, + pub kind_span: Span, + pub alias_span: Span, + /// Whole quoted header ID token, even if it is not a valid UUID. In that + /// case declaration.id is None. Missing/non-string ID values have no span. + pub id_literal_span: Option, +} + +/// A full-file tolerant scan, not a successful parse or validated model. Invalid +/// text remains available. Lexical errors and unmatched braces are facts only: +/// their absence does not establish syntactic or semantic validity. +/// +/// A budget failure returns Err and no partial Outline. Include reachability, +/// project completeness and editor revision are intentionally not inferred here. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Outline<'a> { + pub source: &'a str, + pub tokens: Vec, + pub declarations: Vec, + pub has_lexical_errors: bool, + pub has_unmatched_braces: bool, +} + /// Tolerant highlighting tokens cover the whole input, including incomplete /// strings and unexpected characters. No model is created or modified. pub fn tokenize(source: &str) -> Vec { + tokenize_with_limit(source, usize::MAX).expect("unlimited token scan") +} + +fn tokenize_with_limit(source: &str, max_tokens: usize) -> Result, OutlineLimit> { let mut tokens = Vec::new(); let mut cursor = 0; while cursor < source.len() { + if tokens.len() == max_tokens { + return Err(OutlineLimit { + resource: OutlineResource::Tokens, + limit: max_tokens, + observed_at_least: tokens.len().saturating_add(1), + offset: cursor, + }); + } let start = cursor; let ch = source[cursor..].chars().next().expect("in bounds"); let class = if matches!(ch, ' ' | '\t' | '\r' | '\n') { cursor += 1; @@ -223,9 +298,9 @@ class, span: Span { start, end: cursor }, }); } - tokens + Ok(tokens) } pub fn lint(source: &str) -> Vec { match parse_document(source, ParseLimits::UNLIMITED) { @@ -1229,13 +1304,9 @@ } self.cursor += 1; let offset = self.current().span.start; let value = self.quoted()?; - if value.len() != 36 - || ![8, 13, 18, 23] - .into_iter() - .all(|i| value.as_bytes()[i] == b'-') - { + if !has_uuid_header_shape(&value) { return Err(Error { code: "invalid_source", offset, message: "Expected UUID in 8-4-4-4-12 hexadecimal form.".into(), @@ -1961,29 +2032,74 @@ } /// Analyze an incomplete source buffer without adopting a candidate. Declaration /// spans remain available even when a later declaration contains errors. +/// Invalid compact/braced/URN header IDs retain no authored identity, matching +/// the declaration parser rather than the UUID crate's broader accepted forms. pub fn analyze(source: &str) -> Analysis { let tokens = tokenize(source); + let (declarations, _) = + declaration_outline(source, &tokens, usize::MAX).expect("unlimited declaration walk"); + Analysis { + tokens, + diagnostics: lint(source), + declarations: declarations.into_iter().map(|d| d.declaration).collect(), + } +} + +/// Read exact current text without lint, parsing a model, stabilization or I/O. +/// Source bytes are admitted before token allocation; token/declaration limits +/// are checked before lexing the next token or materializing the next record. +pub fn outline(source: &str, limits: OutlineLimits) -> Result, OutlineLimit> { + if source.len() > limits.max_source_bytes { + return Err(OutlineLimit { + resource: OutlineResource::SourceBytes, + limit: limits.max_source_bytes, + observed_at_least: source.len(), + offset: 0, + }); + } + let tokens = tokenize_with_limit(source, limits.max_tokens)?; + let has_lexical_errors = tokens.iter().any(|token| token.class == TokenClass::Error); + let (declarations, has_unmatched_braces) = + declaration_outline(source, &tokens, limits.max_declarations)?; + Ok(Outline { + source, + tokens, + declarations, + has_lexical_errors, + has_unmatched_braces, + }) +} + +fn declaration_outline( + source: &str, + tokens: &[TokenSpan], + max_declarations: usize, +) -> Result<(Vec, bool), OutlineLimit> { let significant = tokens .iter() .filter(|t| !matches!(t.class, TokenClass::Whitespace | TokenClass::Comment)) .collect::>(); // Recover matched body bounds once. Scanning each aggregate's descendants // again would make tolerant navigation quadratic on deeply nested drafts. let mut body_ends = vec![None; significant.len()]; let mut bodies = Vec::new(); + let mut unmatched_close = false; for (index, token) in significant.iter().enumerate() { match &source[token.span.start..token.span.end] { "{" => bodies.push(index), "}" => { if let Some(open) = bodies.pop() { body_ends[open] = Some(token.span.end); + } else { + unmatched_close = true; } } _ => {} } } + let has_unmatched_braces = unmatched_close || !bodies.is_empty(); let mut declarations = Vec::new(); for (i, t) in significant.iter().enumerate() { let word = &source[t.span.start..t.span.end]; if declaration_kind(word).is_none() && word != "project" { @@ -2006,48 +2122,69 @@ continue; } j += 1; } + if declarations.len() == max_declarations { + return Err(OutlineLimit { + resource: OutlineResource::Declarations, + limit: max_declarations, + observed_at_least: declarations.len().saturating_add(1), + offset: t.span.start, + }); + } let mut id = None; + let mut id_literal_span = None; if significant .get(j) .is_some_and(|t| &source[t.span.start..t.span.end] == "id") { if let Some(value) = significant.get(j + 1) && value.class == TokenClass::String { + id_literal_span = Some(value.span); let text = serde_json::from_str::(&source[value.span.start..value.span.end]) .unwrap(); - id = Id::parse_str(&text).ok(); + if has_uuid_header_shape(&text) { + id = Id::parse_str(&text).ok(); + } } j += 2; } - let mut end = significant - .get(j) - .map_or(alias_token.span.end, |t| t.span.end); + // A recognized header ID at EOF must remain inside the whole span. + // Keep the old next-token/body behavior; only the EOF fallback changes. + let header_end = significant + .get(j.saturating_sub(1)) + .or_else(|| significant.last()) + .map_or(alias_token.span.end, |token| token.span.end); + let mut end = significant.get(j).map_or(header_end, |t| t.span.end); if significant .get(j) .is_some_and(|t| &source[t.span.start..t.span.end] == "{") { end = body_ends[j].unwrap_or(source.len()); } - declarations.push(DeclarationSpan { - alias: alias.into(), - id, - kind: word.into(), - span: Span { - start: t.span.start, - end, + declarations.push(OutlineDeclaration { + declaration: DeclarationSpan { + alias: alias.into(), + id, + kind: word.into(), + span: Span { + start: t.span.start, + end, + }, }, + kind_span: t.span, + alias_span: alias_token.span, + id_literal_span, }); } - Analysis { - tokens, - diagnostics: lint(source), - declarations, - } -} + Ok((declarations, has_unmatched_braces)) +} + +#[cfg(test)] +#[path = "language_outline_tests.rs"] +mod outline_tests; #[cfg(test)] #[path = "language_nesting_tests.rs"] mod nesting_tests; @@ -2441,13 +2578,9 @@ let offset = self.current().offset; let value = self.quoted()?; // Require the portable 8-4-4-4-12 spelling, not the UUID crate's URN, // braces, or compact alternatives. Uppercase hexadecimal is accepted. - if value.len() != 36 - || ![8, 13, 18, 23] - .into_iter() - .all(|index| value.as_bytes()[index] == b'-') - { + if !super::has_uuid_header_shape(&value) { return Err(Error { code: "invalid_source", offset, message: "Expected a UUID string in 8-4-4-4-12 hexadecimal form.".into(), --- /dev/null +++ b/crates/oa-core/src/language_outline_tests.rs @@ -0,0 +1,271 @@ +//! Prepared outline tests. Explicit compatibility tests additionally exercise +//! the existing parsers/analyze; the outline entry itself never calls them. +use super::*; + +fn limits() -> OutlineLimits { + OutlineLimits { + max_source_bytes: 1024 * 1024, + max_tokens: 100_000, + max_declarations: 1_000, + } +} + +fn declaration<'a>(result: &'a Outline<'_>, alias: &str) -> &'a OutlineDeclaration { + result.declarations.iter().find(|d| d.declaration.alias == alias).unwrap() +} + +fn text(source: &str, span: Span) -> &str { + &source[span.start..span.end] +} + +#[test] +fn outline_roles_preserve_unicode_crlf_and_exact_escaped_uuid_literal() { + let source = "// 検査 🚒\r\noa_actor responder \"救助\" id \"\\u00300000000-0000-0000-0000-000000000001\" {}\r\nsource @ // 使用\r\n responder;"; + let before = source.as_bytes().to_vec(); + let result = outline(source, limits()).unwrap(); + assert_eq!(result.source.as_bytes(), before); + assert_eq!(result.source.as_ptr(), source.as_ptr()); + let d = declaration(&result, "responder"); + assert_eq!(text(source, d.kind_span), "oa_actor"); + assert_eq!(text(source, d.alias_span), "responder"); + assert_eq!(text(source, d.id_literal_span.unwrap()), "\"\\u00300000000-0000-0000-0000-000000000001\""); + assert_eq!(d.declaration.id, Some(Id::from_u128(1))); + assert_eq!(d.declaration.span.start, source.find("oa_actor").unwrap()); + assert_eq!(d.declaration.span.end, source.find("{}\r\n").unwrap() + 2); + let mut end = 0; + for token in &result.tokens { + assert_eq!(token.span.start, end); + assert!(token.span.start < token.span.end); + assert!(source.is_char_boundary(token.span.start)); + assert!(source.is_char_boundary(token.span.end)); + end = token.span.end; + } + assert_eq!(end, source.len()); + assert!(!result.has_lexical_errors); + assert!(!result.has_unmatched_braces); +} + +#[test] +fn closed_id_at_eof_is_inside_the_declaration_extent() { + for source in [ + "oa_actor a \"Actor\" id \"00000000-0000-0000-0000-000000000001\"", + "node placed id \"00000000-0000-0000-0000-000000000001\"", + ] { + let result = outline(source, limits()).unwrap(); + let d = &result.declarations[0]; + let literal = d.id_literal_span.unwrap(); + assert_eq!(d.declaration.id, Some(Id::from_u128(1))); + assert_eq!(d.declaration.span.end, source.len()); + assert!(d.declaration.span.start <= literal.start); + assert!(literal.end <= d.declaration.span.end); + } + for source in ["oa_actor a \"Actor\"", "oa_actor a \"Actor\" id"] { + let result = outline(source, limits()).unwrap(); + let d = &result.declarations[0]; + assert_eq!(d.declaration.span.end, source.len()); + assert_eq!(d.declaration.id, None); + assert_eq!(d.id_literal_span, None); + } +} + +#[test] +fn missing_bad_and_historical_ids_never_allocate_authored_identity() { + let source = "oa_actor missing \"No ID\" {}\noa_actor bad \"Bad ID\" id \"not-a-uuid\" {}\noa_actor historical \"00000000-0000-0000-0000-000000000001\" { history uuid \"00000000-0000-0000-0000-000000000001\"; }"; + let first = outline(source, limits()).unwrap(); + let second = outline(source, limits()).unwrap(); + assert_eq!(first, second); + assert!(first.declarations.iter().all(|d| d.declaration.id.is_none())); + assert_eq!(declaration(&first, "missing").id_literal_span, None); + assert_eq!(declaration(&first, "historical").id_literal_span, None); + assert_eq!(text(source, declaration(&first, "bad").id_literal_span.unwrap()), "\"not-a-uuid\""); + // Balanced tokens are not a successful syntax/semantic validation claim. + assert!(!first.has_lexical_errors); + assert!(!first.has_unmatched_braces); +} + +#[test] +fn invalid_full_scan_keeps_closed_children_and_reports_lexical_body_facts() { + let source = "oa_entity outer \"Outer\" {\n oa_actor ready \"Ready\" { description \"} stays text\"; }\n oa_actor pending \"Pending\" { description \"unfinished 🌊"; + let result = outline(source, limits()).unwrap(); + assert_eq!(result.source, source); + assert_eq!(result.tokens.last().unwrap().span.end, source.len()); + assert!(result.has_lexical_errors); + assert!(result.has_unmatched_braces); + let ready = declaration(&result, "ready"); + assert_eq!(text(source, ready.declaration.span), "oa_actor ready \"Ready\" { description \"} stays text\"; }"); + assert_eq!(declaration(&result, "outer").declaration.span.end, source.len()); + assert_eq!(declaration(&result, "pending").declaration.span.end, source.len()); + let unmatched_close = outline("}\n// brace in comment {\n", limits()).unwrap(); + assert!(unmatched_close.has_unmatched_braces); + assert!(!unmatched_close.has_lexical_errors); +} + +#[test] +fn missing_body_syntax_is_unchecked_even_when_known_error_flags_are_clear() { + let source = "oa_actor a \"A\" id \"00000000-0000-0000-0000-000000000001\""; + let result = outline(source, limits()).unwrap(); + assert_eq!(result.declarations.len(), 1); + assert!(!result.has_lexical_errors); + assert!(!result.has_unmatched_braces); + // There is deliberately no `valid` flag: the known-complete fact is scanning + // these bytes, not the missing body becoming legal or the draft being applied. + assert_eq!(result.source, source); +} + +#[test] +fn keyword_alias_and_collisions_keep_every_source_ordered_header() { + let source = "project root_model \"Model\" {\n oa_actor oa_activity \"First\" id \"00000000-0000-0000-0000-000000000001\" {}\n oa_actor oa_activity \"Second\" id \"00000000-0000-0000-0000-000000000001\" {}\n diagram view_one \"View\" {}\n node placement { diagram @view_one; }\n}"; + let result = outline(source, limits()).unwrap(); + assert_eq!(result.declarations.len(), 5); + let duplicates: Vec<_> = result.declarations.iter().filter(|d| d.declaration.alias == "oa_activity").collect(); + assert_eq!(duplicates.len(), 2); + assert!(duplicates[0].alias_span.start < duplicates[1].alias_span.start); + assert_eq!(duplicates[0].declaration.id, duplicates[1].declaration.id); + assert!(result.tokens.iter().any(|t| t.span == duplicates[0].alias_span && t.class == TokenClass::Keyword)); + assert_eq!(text(source, declaration(&result, "placement").kind_span), "node"); + assert_eq!(declaration(&result, "placement").declaration.id, None); +} + +#[test] +fn bounds_refuse_before_returning_a_partial_outline() { + let source = "oa_actor a \"A\" {}\noa_actor b \"B\" {}"; + let admitted = outline(source, limits()).unwrap(); + let exact = OutlineLimits { + max_source_bytes: source.len(), + max_tokens: admitted.tokens.len(), + max_declarations: 2, + }; + assert_eq!(outline(source, exact).unwrap(), admitted); + for resource in [OutlineResource::SourceBytes, OutlineResource::Tokens, OutlineResource::Declarations] { + let mut lower = exact; + match resource { + OutlineResource::SourceBytes => lower.max_source_bytes -= 1, + OutlineResource::Tokens => lower.max_tokens -= 1, + OutlineResource::Declarations => lower.max_declarations -= 1, + } + let error = outline(source, lower).unwrap_err(); + assert_eq!(error.resource, resource); + assert!(error.observed_at_least > error.limit); + assert!(source.is_char_boundary(error.offset)); + if resource == OutlineResource::Declarations { + assert_eq!(error.offset, source.find("oa_actor b").unwrap()); + } + } + let empty = outline("", OutlineLimits { max_source_bytes: 0, max_tokens: 0, max_declarations: 0 }).unwrap(); + assert!(empty.tokens.is_empty() && empty.declarations.is_empty()); + let unicode = "// 🌊\r\noa_actor a \"A\" {}"; + let error = outline(unicode, OutlineLimits { max_source_bytes: 4, ..limits() }).unwrap_err(); + assert_eq!(error.resource, OutlineResource::SourceBytes); + assert_eq!(error.offset, 0); + let error = outline(unicode, OutlineLimits { max_tokens: 1, ..limits() }).unwrap_err(); + assert_eq!(error.resource, OutlineResource::Tokens); + assert_eq!(error.offset, unicode.find('\n').unwrap()); +} + +#[test] +fn outline_ranges_follow_current_bytes_without_identity_stabilization() { + let old = "oa_actor a \"A\" {}\nsource @a;"; + let current = format!("// 航海 🚒\r\n{old}"); + let old_outline = outline(old, limits()).unwrap(); + let current_outline = outline(¤t, limits()).unwrap(); + let shift = current.len() - old.len(); + let old_role = &old_outline.declarations[0]; + let current_role = ¤t_outline.declarations[0]; + assert_eq!(current_role.alias_span.start, old_role.alias_span.start + shift); + assert_eq!(current_role.kind_span.start, old_role.kind_span.start + shift); + assert_eq!(current_role.declaration.span.end, old_role.declaration.span.end + shift); + assert_eq!(current_outline.source, current); + assert_eq!(old_outline.source, old); + assert_eq!(current_role.declaration.id, None); + assert!(!current_outline.source.contains(" id ")); + // Workspace revision/stale-handle rejection belongs to the query/adapter, + // not a claim that an old owned span can be applied to any other string. +} + +#[test] +fn deeply_nested_bounds_use_the_same_iterative_walk_and_declaration_budget() { + let mut source = String::new(); + for index in 0..600 { + source.push_str(&format!("oa_activity work_{index} \"Work\" {{\n")); + } + source.push_str(&"}\n".repeat(600)); + let result = outline(&source, limits()).unwrap(); + assert_eq!(result.declarations.len(), 600); + assert!(!result.has_unmatched_braces); + assert_eq!(result.declarations[0].declaration.span.end, source.len() - 1); + let error = outline(&source, OutlineLimits { max_declarations: 599, ..limits() }).unwrap_err(); + assert_eq!(error.resource, OutlineResource::Declarations); + assert_eq!(error.observed_at_least, 600); + assert_eq!(error.offset, source.find("oa_activity work_599").unwrap()); +} + +#[test] +fn noncanonical_uuid_spellings_have_no_outline_or_analysis_identity() { + for raw in [ + "00000000000000000000000000000001", + "{00000000-0000-0000-0000-000000000001}", + "urn:uuid:00000000-0000-0000-0000-000000000001", + ] { + assert!(Id::parse_str(raw).is_ok(), "The UUID crate alone is too permissive"); + assert!(!has_uuid_header_shape(raw)); + let literal = format!("\"{raw}\""); + let modern = format!("arcadia 7; project model \"Model\" id {literal} {{}}"); + let before = modern.clone(); + let result = outline(&modern, limits()).unwrap(); + let d = declaration(&result, "model"); + assert_eq!(d.declaration.id, None); + assert_eq!(text(&modern, d.id_literal_span.unwrap()), literal); + assert_eq!(modern, before); + let errors = parse(&modern).unwrap_err(); + assert!(errors.iter().any(|error| error.message.contains("8-4-4-4-12") + && error.span.start == modern.find(&literal).unwrap())); + // Intentional compatibility correction: old analyze yielded Some(Id(1)) + // here despite lint rejecting the header. It now shares outline's None. + let analysis = analyze(&modern); + assert_eq!(analysis.declarations.iter().find(|d| d.alias == "model").unwrap().id, None); + assert!(analysis.diagnostics.iter().any(|error| error.message.contains("8-4-4-4-12"))); + let legacy_source = format!("arcadia 1; project \"Legacy\" id {literal} {{}}"); + let legacy_errors = legacy::parse(&legacy_source).unwrap_err(); + assert!(legacy_errors.iter().any(|error| error.message.contains("8-4-4-4-12"))); + } +} + +#[test] +fn uppercase_and_json_escaped_header_ids_survive_both_parsers_and_outline() { + let expected = Id::from_u128(0xabcdef); + let canonical = expected.to_string(); + let uppercase = canonical.to_uppercase(); + assert_ne!(uppercase, canonical); + for raw in [uppercase.clone(), format!("\\u0030{}", &uppercase[1..])] { + let literal = format!("\"{raw}\""); + let decoded: String = serde_json::from_str(&literal).unwrap(); + assert!(has_uuid_header_shape(&decoded)); + let modern = format!("arcadia 7; project model \"Model\" id {literal} {{}}"); + let result = outline(&modern, limits()).unwrap(); + let d = declaration(&result, "model"); + assert_eq!(d.declaration.id, Some(expected)); + assert_eq!(text(&modern, d.id_literal_span.unwrap()), literal); + assert_eq!(result.source.as_bytes(), modern.as_bytes()); + assert_eq!(parse(&modern).unwrap().id, expected); + let analysis = analyze(&modern); + assert_eq!(analysis.declarations[0].id, Some(expected)); + assert!(!analysis.diagnostics.iter().any(|error| error.severity == crate::validation::Severity::Error)); + let legacy_source = format!("arcadia 1; project \"Legacy\" id {literal} {{}}"); + assert_eq!(legacy::parse(&legacy_source).unwrap().id, expected); + } +} + +#[test] +fn header_shape_still_requires_valid_hexadecimal_uuid_data() { + let raw = "00000000-0000-0000-0000-00000000000Z"; + assert!(has_uuid_header_shape(raw)); + assert!(Id::parse_str(raw).is_err()); + let literal = format!("\"{raw}\""); + let source = format!("arcadia 7; project model \"Model\" id {literal} {{}}"); + let result = outline(&source, limits()).unwrap(); + let d = declaration(&result, "model"); + assert_eq!(d.declaration.id, None); + assert_eq!(text(&source, d.id_literal_span.unwrap()), literal); + assert!(parse(&source).unwrap_err().iter().any(|error| error.message == "Invalid UUID identity.")); + assert_eq!(analyze(&source).declarations[0].id, None); +}