Add pure modular source3 project compiler and update APIs

crates/oa-core/src/language.rs

@@ -219,12 +219,12 @@ impl fmt::Display for Diagnostic {
 }
 
 #[derive(Debug)]
-struct Error {
-    code: &'static str,
-    offset: usize,
-    message: String,
+pub(crate) struct Error {
+    pub(crate) code: &'static str,
+    pub(crate) offset: usize,
+    pub(crate) message: String,
 }
-fn diagnostic(source: &str, error: Error) -> Diagnostic {
+pub(crate) fn diagnostic(source: &str, error: Error) -> Diagnostic {
     let start = error.offset.min(source.len());
     let prefix = &source[..start];
     Diagnostic {
@@ -244,7 +244,7 @@ fn diagnostic(source: &str, error: Error) -> Diagnostic {
 fn reserved(word: &str) -> bool {
     crate::reserved_alias(word)
         || declaration_kind(word).is_some()
-        || matches!(word, "none" | "node")
+        || matches!(word, "none" | "node" | "include")
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -534,11 +534,6 @@ fn offset_for(source: &str, line: usize, column: usize) -> usize {
 /// Canonical output uses persistent aliases and explicit identities. Typed
 /// nested records are ordinary language fields; JSON documents are not embedded.
 pub fn format(project: &Project) -> String {
-    let aliases = project
-        .identities()
-        .into_iter()
-        .map(|(id, a)| (id, a.to_owned()))
-        .collect::<BTreeMap<_, _>>();
     let mut out = std::format!(
         "arcadia 3;\nproject {} {} id \"{}\" {{\n    description {};\n\n",
         project.alias,
@@ -546,7 +541,23 @@ pub fn format(project: &Project) -> String {
         project.id,
         quote(&project.description)
     );
+    for (_, declaration) in format_declarations(project) {
+        out.push_str(&declaration);
+    }
+    out.push_str("}\n");
+    out
+}
+
+/// The same typed declaration writer is used by flat and modular source output.
+pub(crate) fn format_declarations(project: &Project) -> Vec<(Id, String)> {
+    let aliases = project
+        .identities()
+        .into_iter()
+        .map(|(id, alias)| (id, alias.to_owned()))
+        .collect();
+    let mut declarations = Vec::new();
     for e in project.elements.values() {
+        let mut out = String::new();
         let mut fields = payload(&serde_json::to_value(&e.body).unwrap());
         fields.insert("description".into(), Value::String(e.description.clone()));
         fields.insert("parent".into(), serde_json::to_value(e.parent).unwrap());
@@ -559,8 +570,10 @@ pub fn format(project: &Project) -> String {
             &fields,
             &aliases,
         );
+        declarations.push((e.id, out));
     }
     for r in project.relations.values() {
+        let mut out = String::new();
         let mut fields = payload(&serde_json::to_value(&r.body).unwrap());
         fields.insert("description".into(), Value::String(r.description.clone()));
         write_declaration(
@@ -572,8 +585,10 @@ pub fn format(project: &Project) -> String {
             &fields,
             &aliases,
         );
+        declarations.push((r.id, out));
     }
     for d in project.diagrams.values() {
+        let mut out = String::new();
         let mut fields = record_fields(d);
         fields.remove("nodes");
         write_declaration(
@@ -585,13 +600,17 @@ pub fn format(project: &Project) -> String {
             &fields,
             &aliases,
         );
+        declarations.push((d.id, out));
         for n in d.nodes.values() {
+            let mut out = String::new();
             let mut fields = record_fields(n);
             fields.insert("diagram".into(), Value::String(d.id.to_string()));
             write_declaration(&mut out, "node", &n.alias, None, n.id, &fields, &aliases);
+            declarations.push((n.id, out));
         }
     }
     for a in project.artifact_links.values() {
+        let mut out = String::new();
         write_declaration(
             &mut out,
             "artifact",
@@ -601,8 +620,10 @@ pub fn format(project: &Project) -> String {
             &record_fields(a),
             &aliases,
         );
+        declarations.push((a.id, out));
     }
     for t in project.traces.values() {
+        let mut out = String::new();
         write_declaration(
             &mut out,
             "trace",
@@ -612,10 +633,11 @@ pub fn format(project: &Project) -> String {
             &record_fields(t),
             &aliases,
         );
+        declarations.push((t.id, out));
     }
-    out.push_str("}\n");
-    out
+    declarations
 }
+
 fn record_fields<T: serde::Serialize>(value: &T) -> Map<String, Value> {
     let mut fields = serde_json::to_value(value)
         .unwrap()
@@ -867,13 +889,49 @@ fn strict_tokens(source: &str) -> Result<Vec<Lexeme>, Error> {
     });
     Ok(out)
 }
-struct Declaration {
+pub(crate) struct Declaration {
     kind: DeclKind,
-    alias: String,
+    pub(crate) alias: String,
     name: String,
-    id: Id,
+    pub(crate) id: Id,
     fields: BTreeMap<String, Ast>,
     offset: usize,
+    pub(crate) span: Span,
+    pub(crate) id_insertion: Option<usize>,
+    pub(crate) field_spans: BTreeMap<String, Span>,
+}
+impl Declaration {
+    pub(crate) fn is_graph_object(&self) -> bool {
+        matches!(self.kind, DeclKind::Element(..) | DeclKind::Relation(..))
+    }
+}
+pub(crate) struct SourceHeader {
+    pub(crate) project: Project,
+    pub(crate) span: Span,
+    pub(crate) id_insertion: Option<usize>,
+    pub(crate) description: Option<Span>,
+    pub(crate) body: Span,
+}
+pub(crate) struct SourceUnit {
+    pub(crate) header: Option<SourceHeader>,
+    pub(crate) declarations: Vec<Declaration>,
+    pub(crate) includes: Vec<(String, Span)>,
+}
+pub(crate) fn parse_unit(
+    source: &str,
+    entry: bool,
+    limits: ParseLimits,
+) -> Result<SourceUnit, Box<Diagnostic>> {
+    let tokens = strict_tokens(source).map_err(|e| Box::new(diagnostic(source, e)))?;
+    Parser {
+        source,
+        tokens,
+        cursor: 0,
+        value_depth: 0,
+        limits,
+    }
+    .unit(entry, true)
+    .map_err(|e| Box::new(diagnostic(source, e)))
 }
 struct Parser<'a> {
     source: &'a str,
@@ -1029,59 +1087,140 @@ impl Parser<'_> {
         }
     }
     fn document(&mut self) -> Result<ParsedDocument, Error> {
-        self.take("arcadia")?;
-        if self.text() != "3" {
-            return Err(self.error("Unsupported schema version: expected 'arcadia 3;'."));
+        let unit = self.unit(true, false)?;
+        let header = unit.header.expect("entry header");
+        let mut aliases = BTreeMap::from([(header.project.alias.clone(), header.project.id)]);
+        let mut locations = BTreeMap::from([(header.project.id, header.span.start)]);
+        let mut fields = BTreeMap::new();
+        for d in &unit.declarations {
+            if aliases.insert(d.alias.clone(), d.id).is_some() {
+                return Err(Error {
+                    code: "invalid_source",
+                    offset: d.offset,
+                    message: format!("Duplicate alias '{}'.", d.alias),
+                });
+            }
+            if locations.insert(d.id, d.offset).is_some() {
+                return Err(Error {
+                    code: "invalid_source",
+                    offset: d.offset,
+                    message: format!("Duplicate identity '{}'.", d.id),
+                });
+            }
+            fields.extend(
+                d.field_spans
+                    .iter()
+                    .map(|(key, span)| ((d.id, key.clone()), span.start)),
+            );
         }
-        self.cursor += 1;
-        self.take(";")?;
-        self.take("project")?;
-        let alias = self.identifier()?;
-        let name = self.quoted()?;
-        let id = self.optional_id()?;
-        self.take("{")?;
-        let mut project = Project {
-            schema_version: SCHEMA_VERSION,
-            id,
-            alias: alias.clone(),
-            name,
-            description: String::new(),
-            elements: BTreeMap::new(),
-            relations: BTreeMap::new(),
-            diagrams: BTreeMap::new(),
-            artifact_links: BTreeMap::new(),
-            traces: BTreeMap::new(),
+        let project =
+            resolve_declarations(header.project, &unit.declarations, &aliases, self.limits)
+                .map_err(|(_, error)| error)?;
+        Ok(ParsedDocument {
+            project,
+            migration: None,
+            locations,
+            fields,
+        })
+    }
+    fn unit(&mut self, entry: bool, includes_allowed: bool) -> Result<SourceUnit, Error> {
+        let mut header = if entry {
+            self.take("arcadia")?;
+            if self.text() != "3" {
+                return Err(self.error("Unsupported schema version: expected 'arcadia 3;'."));
+            }
+            self.cursor += 1;
+            self.take(";")?;
+            let start = self.current().span.start;
+            self.take("project")?;
+            let alias = self.identifier()?;
+            let name = self.quoted()?;
+            let id_insertion = (self.text() != "id").then_some(self.current().span.start);
+            let id = self.optional_id()?;
+            let end = self.current().span.start;
+            self.take("{")?;
+            Some(SourceHeader {
+                project: Project {
+                    schema_version: SCHEMA_VERSION,
+                    id,
+                    alias,
+                    name,
+                    description: String::new(),
+                    elements: BTreeMap::new(),
+                    relations: BTreeMap::new(),
+                    diagrams: BTreeMap::new(),
+                    artifact_links: BTreeMap::new(),
+                    traces: BTreeMap::new(),
+                },
+                span: Span { start, end },
+                id_insertion,
+                description: None,
+                body: Span {
+                    start: end + 1,
+                    end: end + 1,
+                },
+            })
+        } else {
+            None
         };
-        let mut aliases = BTreeMap::from([(alias, id)]);
-        let mut locations = BTreeMap::from([(id, 0)]);
         let mut declarations = Vec::new();
+        let mut includes = Vec::new();
         let mut graph_count = 0usize;
-        let mut field_locations = BTreeMap::new();
-        let mut has_description = false;
-        while self.text() != "}" {
-            if self.text() == "description" {
-                if has_description {
-                    return Err(self.error("Duplicate project description."));
+        while if entry {
+            self.text() != "}"
+        } else {
+            !self.text().is_empty()
+        } {
+            let offset = self.current().span.start;
+            if self.text() == "include" {
+                if !includes_allowed {
+                    return Err(self.error("An include requires a source bundle; single-string parsing never reads files."));
                 }
-                has_description = true;
                 self.cursor += 1;
-                project.description = self.quoted()?;
+                let path = self.quoted()?;
                 self.take(";")?;
+                includes.push((
+                    path,
+                    Span {
+                        start: offset,
+                        end: self.tokens[self.cursor - 1].span.end,
+                    },
+                ));
+                continue;
+            }
+            if self.text() == "description"
+                && let Some(mut entry_header) = header.take()
+            {
+                if entry_header.description.is_some() {
+                    return Err(self.error("Duplicate project description."));
+                }
+                self.cursor += 1;
+                let description = self.quoted()?;
+                self.take(";")?;
+                entry_header.project.description = description;
+                entry_header.description = Some(Span {
+                    start: offset,
+                    end: self.tokens[self.cursor - 1].span.end,
+                });
+                header = Some(entry_header);
                 continue;
             }
-            let offset = self.current().span.start;
             let word = self.text().to_owned();
             let kind = declaration_kind(&word)
-                .ok_or_else(|| self.error(std::format!("Unknown declaration '{word}'.")))?;
+                .ok_or_else(|| self.error(format!("Unknown declaration '{word}'.")))?;
             graph_count += usize::from(matches!(
                 kind,
                 DeclKind::Element(..) | DeclKind::Relation(..)
             ));
             self.limits
-                .check_counts(graph_count, declarations.len().saturating_add(2), 0)
+                .check_counts(
+                    graph_count,
+                    declarations.len().saturating_add(1 + usize::from(entry)),
+                    0,
+                )
                 .map_err(|message| Error {
                     code: "model_limit",
-                    offset: self.current().span.start,
+                    offset,
                     message,
                 })?;
             self.cursor += 1;
@@ -1091,25 +1230,29 @@ impl Parser<'_> {
             } else {
                 self.quoted()?
             };
+            let id_insertion = (self.text() != "id").then_some(self.current().span.start);
             let id = self.optional_id()?;
             let fields = self.fields()?;
-            for (field, value) in &fields {
-                field_locations.insert((id, field.clone()), value.offset());
-            }
-            if aliases.insert(alias.clone(), id).is_some() {
-                return Err(Error {
-                    code: "invalid_source",
-                    offset,
-                    message: std::format!("Duplicate alias '{alias}'."),
-                });
-            }
-            if locations.insert(id, offset).is_some() {
-                return Err(Error {
-                    code: "invalid_source",
-                    offset,
-                    message: std::format!("Duplicate identity '{id}'."),
-                });
-            }
+            let field_spans = fields
+                .iter()
+                .map(|(key, value)| {
+                    let start = value.offset();
+                    let mut depth = 0usize;
+                    let mut end = start;
+                    for token in
+                        self.tokens[self.tokens.partition_point(|t| t.span.start < start)..].iter()
+                    {
+                        match token.text(self.source) {
+                            ";" if depth == 0 => break,
+                            "{" | "[" => depth += 1,
+                            "}" | "]" => depth = depth.saturating_sub(1),
+                            _ => {}
+                        }
+                        end = token.span.end;
+                    }
+                    (key.clone(), Span { start, end })
+                })
+                .collect();
             declarations.push(Declaration {
                 kind,
                 alias,
@@ -1117,20 +1260,44 @@ impl Parser<'_> {
                 id,
                 fields,
                 offset,
+                span: Span {
+                    start: offset,
+                    end: self.tokens[self.cursor - 1].span.end,
+                },
+                id_insertion,
+                field_spans,
             });
         }
-        self.take("}")?;
+        if entry {
+            header.as_mut().expect("entry header").body.end = self.current().span.start;
+            self.take("}")?;
+        }
         if !self.text().is_empty() {
             return Err(self.error("Unexpected content after the project closing brace."));
         }
-        let mut nodes = Vec::new();
-        for declaration in declarations {
+        Ok(SourceUnit {
+            header,
+            declarations,
+            includes,
+        })
+    }
+}
+
+pub(crate) fn resolve_declarations(
+    mut project: Project,
+    declarations: &[Declaration],
+    aliases: &BTreeMap<String, Id>,
+    limits: ParseLimits,
+) -> Result<Project, (usize, Error)> {
+    let mut nodes = Vec::new();
+    for (index, declaration) in declarations.iter().enumerate() {
+        (|| -> Result<(), Error> {
             let mut fields = declaration
                 .fields
                 .iter()
-                .map(|(k, v)| Ok((k.clone(), v.resolve(k, &aliases)?)))
+                .map(|(k, v)| Ok((k.clone(), v.resolve(k, aliases)?)))
                 .collect::<Result<Map<_, _>, Error>>()?;
-            let d = &declaration;
+            let d = declaration;
             let conversion_error = |message: String| Error {
                 code: "invalid_source",
                 offset: d
@@ -1249,7 +1416,7 @@ impl Parser<'_> {
                     header(&mut record, d, false);
                     let node: NodePlacement = serde_json::from_value(Value::Object(record))
                         .map_err(|e| conversion_error(e.to_string()))?;
-                    nodes.push((diagram, node, d.offset));
+                    nodes.push((diagram, node, d.offset, index));
                 }
                 DeclKind::Artifact => {
                     if !fields.contains_key("owner") {
@@ -1280,28 +1447,34 @@ impl Parser<'_> {
                     project.traces.insert(d.id, trace);
                 }
             }
-        }
-        for (diagram, node, offset) in nodes {
-            let target = project.diagrams.get_mut(&diagram).ok_or_else(|| Error {
+            Ok(())
+            })().map_err(|error| (index, error))?;
+    }
+    for (diagram, node, offset, index) in nodes {
+        let target = project
+            .diagrams
+            .get_mut(&diagram)
+            .ok_or_else(|| Error {
                 code: "invalid_source",
                 offset,
                 message: "Node 'diagram' must reference a diagram declaration.".into(),
-            })?;
-            target.nodes.insert(node.id, node);
-        }
-        self.limits.check_model(&project).map_err(|message| Error {
-            code: "model_limit",
-            offset: 0,
-            message,
-        })?;
-        Ok(ParsedDocument {
-            project,
-            migration: None,
-            locations,
-            fields: field_locations,
-        })
+            })
+            .map_err(|error| (index, error))?;
+        target.nodes.insert(node.id, node);
     }
+    limits.check_model(&project).map_err(|message| {
+        (
+            usize::MAX,
+            Error {
+                code: "model_limit",
+                offset: 0,
+                message,
+            },
+        )
+    })?;
+    Ok(project)
 }
+
 fn header(fields: &mut Map<String, Value>, d: &Declaration, named: bool) {
     fields.insert("id".into(), Value::String(d.id.to_string()));
     fields.insert("alias".into(), Value::String(d.alias.clone()));