feat: preserve nested source projects across desktop and REST

2026-09-06 00:17:20 +03:00
parent d364d3c058
commit 4e8575bd11
32 changed files with 4283 additions and 417 deletions
@@ -1,7 +1,8 @@
 //! Single-owner on-disk envelopes. All methods run on a blocking worker.
-use crate::error::{ApiError, ApiResult, issues};
+use crate::error::{ApiError, ApiResult, issues, source_error};
 use axum::http::StatusCode;
 use oa_core::persistence::{DecodeResult, MigrationReport};
+use oa_core::source_project::{self, CompiledSourceProject, ProjectLimits, SourceBundle};
 use oa_core::{Id, Project};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -13,6 +14,7 @@ use std::{
 };

 pub const MAX_MODEL_BYTES: usize = 8 * 1024 * 1024;
+pub const MAX_SOURCE_PROJECT_BYTES: usize = 8 * 1024 * 1024;
 pub const MAX_OBJECTS: usize = 5_000;
 pub const MAX_PROJECTS: usize = 128;
 pub const MAX_STORE_BYTES: usize = 256 * 1024 * 1024;
@@ -22,7 +24,13 @@ pub(crate) const MODEL_LIMITS: oa_core::language::ParseLimits = oa_core::languag
    max_records: 20_000,
    max_step_references: 50_000,
 };
-const MAX_ENVELOPE_BYTES: usize = MAX_MODEL_BYTES + 512;
+pub(crate) const SOURCE_LIMITS: ProjectLimits = ProjectLimits {
+    max_files: 256,
+    max_include_depth: 32,
+    max_include_edges: 1_024,
+    model: MODEL_LIMITS,
+};
+const MAX_ENVELOPE_BYTES: usize = MAX_MODEL_BYTES + MAX_SOURCE_PROJECT_BYTES + 512;

 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -30,6 +38,7 @@ pub struct StoredProject {
    pub resource_id: Id,
    pub revision: u64,
    pub model: Project,
+    pub source_project: SourceBundle,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub migration: Option<MigrationReport>,
 }
@@ -93,11 +102,88 @@ struct StoreInner {

 /// Constructed only after complete candidate validation, outside the catalog
 /// mutex. Commit methods cannot accidentally accept an unchecked candidate.
-struct PreparedModel(Project, Option<MigrationReport>);
-impl PreparedModel {
+pub(crate) struct PreparedProject {
+    compiled: CompiledSourceProject,
+    migration: Option<MigrationReport>,
+}
+impl PreparedProject {
    fn new(model: Project) -> ApiResult<Self> {
        validate_model(&model)?;
-        Ok(Self(model, None))
+        let prepared = Self::from_bundle(source_project::format_bundle(&model))?;
+        if prepared.model() != &model {
+            return Err(ApiError::internal(
+                "Generated source does not reproduce the model.",
+            ));
+        }
+        Ok(prepared)
+    }
+
+    pub(crate) fn from_bundle(bundle: SourceBundle) -> ApiResult<Self> {
+        check_source_size(&bundle)?;
+        let compiled =
+            source_project::compile_bundle(&bundle, SOURCE_LIMITS).map_err(source_error)?;
+        Self::from_compiled(compiled)
+    }
+
+    fn from_compiled(compiled: CompiledSourceProject) -> ApiResult<Self> {
+        check_model_size(&compiled.decoded.project)?;
+        check_source_size(&compiled.bundle)?;
+        Ok(Self {
+            compiled,
+            migration: None,
+        })
+    }
+
+    pub(crate) fn from_command(
+        current: &StoredProject,
+        prepared: oa_core::PreparedCommand,
+    ) -> ApiResult<Self> {
+        check_model_size(&prepared.project)?;
+        let compiled = source_project::compile_bundle(&current.source_project, SOURCE_LIMITS)
+            .map_err(source_error)?;
+        // Model equality alone is not a source no-op: delete/recreate and
+        // identity swaps can change declaration ownership without changing the
+        // final semantic snapshot.
+        let compiled = source_project::update_bundle_with_identity_map(
+            &compiled,
+            &prepared.project,
+            &prepared.identity_map,
+            SOURCE_LIMITS,
+        )
+        .map_err(source_error)?;
+        Self::from_compiled(compiled)
+    }
+
+    fn replacement(current: &StoredProject, decoded: DecodeResult) -> ApiResult<Self> {
+        validate_model(&decoded.project)?;
+        let compiled = source_project::compile_bundle(&current.source_project, SOURCE_LIMITS)
+            .map_err(source_error)?;
+        let compiled = source_project::update_bundle(&compiled, &decoded.project, SOURCE_LIMITS)
+            .map_err(source_error)?;
+        let mut prepared = Self::from_compiled(compiled)?;
+        prepared.migration = decoded.migration;
+        Ok(prepared)
+    }
+
+    pub(crate) fn model(&self) -> &Project {
+        &self.compiled.decoded.project
+    }
+
+    pub(crate) fn bundle(&self) -> &SourceBundle {
+        &self.compiled.bundle
+    }
+
+    pub(crate) fn source_issues(&self) -> Vec<crate::error::IssueDto> {
+        self.compiled
+            .diagnostics
+            .iter()
+            .cloned()
+            .map(crate::error::source_issue)
+            .collect()
+    }
+
+    pub(crate) fn changed(&self, current: &StoredProject) -> bool {
+        self.model() != &current.model || self.bundle() != &current.source_project
    }
 }

@@ -130,13 +216,17 @@ impl Store {
    }

    pub(crate) fn create(&self, model: Project) -> ApiResult<StoredProject> {
-        let prepared = PreparedModel::new(model)?;
+        let prepared = PreparedProject::new(model)?;
        self.lock()?.create(prepared)
    }

    pub(crate) fn create_decoded(&self, decoded: DecodeResult) -> ApiResult<StoredProject> {
-        let mut prepared = PreparedModel::new(decoded.project)?;
-        prepared.1 = decoded.migration;
+        let mut prepared = PreparedProject::new(decoded.project)?;
+        prepared.migration = decoded.migration;
+        self.lock()?.create(prepared)
+    }
+    pub(crate) fn create_source_project(&self, bundle: SourceBundle) -> ApiResult<StoredProject> {
+        let prepared = PreparedProject::from_bundle(bundle)?;
        self.lock()?.create(prepared)
    }
    pub(crate) fn replace_decoded(
@@ -145,8 +235,8 @@ impl Store {
        expected: &str,
        decoded: DecodeResult,
    ) -> ApiResult<(StoredProject, bool)> {
-        let mut prepared = PreparedModel::new(decoded.project)?;
-        prepared.1 = decoded.migration;
+        let current = self.checked(id, expected)?;
+        let prepared = PreparedProject::replacement(&current, decoded)?;
        self.lock()?.replace(id, expected, prepared)
    }

@@ -154,9 +244,8 @@ impl Store {
        &self,
        id: Id,
        expected: &str,
-        model: Project,
+        prepared: PreparedProject,
    ) -> ApiResult<(StoredProject, bool)> {
-        let prepared = PreparedModel::new(model)?;
        self.lock()?.replace(id, expected, prepared)
    }

@@ -216,8 +305,7 @@ impl Store {
            if bytes.len() > MAX_ENVELOPE_BYTES {
                return Err(corrupt("Envelope exceeds the size limit.".into()));
            }
-            total_bytes = total_bytes.saturating_add(bytes.len());
-            if projects.len() >= MAX_PROJECTS || total_bytes > MAX_STORE_BYTES {
+            if projects.len() >= MAX_PROJECTS {
                return Err(StoreError::Capacity(
                    "At most 128 models and 256 MiB of serialized envelopes are supported.".into(),
                ));
@@ -228,6 +316,8 @@ impl Store {
                resource_id: Id,
                revision: u64,
                model: Box<serde_json::value::RawValue>,
+                #[serde(default, deserialize_with = "present_source_project")]
+                source_project: Option<SourceBundle>,
            }
            let envelope: Envelope =
                serde_json::from_slice(&bytes).map_err(|error| corrupt(error.to_string()))?;
@@ -236,10 +326,27 @@ impl Store {
                MODEL_LIMITS,
            )
            .map_err(|error| corrupt(error.to_string()))?;
+            let prepared = if let Some(bundle) = envelope.source_project {
+                let prepared = PreparedProject::from_bundle(bundle.clone())
+                    .map_err(|error| corrupt(error.detail))?;
+                if prepared.bundle() != &bundle || prepared.model() != &decoded.project {
+                    return Err(corrupt(
+                        "Stored source and model disagree, or source lacks stable identities."
+                            .into(),
+                    ));
+                }
+                prepared
+            } else {
+                // Old model-only envelopes gain deterministic source in memory.
+                // Opening the store never saves or advances their revision.
+                PreparedProject::new(decoded.project.clone())
+                    .map_err(|error| corrupt(error.detail))?
+            };
            let stored = StoredProject {
                resource_id: envelope.resource_id,
                revision: envelope.revision,
                model: decoded.project,
+                source_project: prepared.compiled.bundle,
                migration: decoded.migration,
            };
            if stored.resource_id != filename_id || stored.revision == 0 {
@@ -247,8 +354,18 @@ impl Store {
                    "Resource ID/filename mismatch or zero revision.".into(),
                ));
            }
-            validate_model(&stored.model).map_err(|error| corrupt(error.detail))?;
-            byte_sizes.insert(stored.resource_id, bytes.len());
+            let size = envelope_bytes(&stored)
+                .map_err(|error| corrupt(error.detail))?
+                .len();
+            // Capacity charges the complete accepted pair, including an old
+            // envelope's in-memory source upgrade before its next changed save.
+            total_bytes = total_bytes.saturating_add(size);
+            if total_bytes > MAX_STORE_BYTES {
+                return Err(StoreError::Capacity(
+                    "Complete accepted envelopes exceed 256 MiB.".into(),
+                ));
+            }
+            byte_sizes.insert(stored.resource_id, size);
            if projects.insert(stored.resource_id, stored).is_some() {
                return Err(corrupt("Duplicate hosted resource ID.".into()));
            }
@@ -284,16 +401,16 @@ impl StoreInner {
        Ok(current)
    }

-    fn create(&mut self, prepared: PreparedModel) -> ApiResult<StoredProject> {
+    fn create(&mut self, prepared: PreparedProject) -> ApiResult<StoredProject> {
        if self.projects.len() >= MAX_PROJECTS {
            return Err(capacity_error());
        }
-        let model = prepared.0;
        let stored = StoredProject {
            resource_id: Id::new_v4(),
            revision: 1,
-            model,
-            migration: prepared.1,
+            model: prepared.compiled.decoded.project,
+            source_project: prepared.compiled.bundle,
+            migration: prepared.migration,
        };
        let size = self.write(&stored, true)?;
        self.adopt(stored.clone(), size);
@@ -305,14 +422,13 @@ impl StoreInner {
        &mut self,
        id: Id,
        expected: &str,
-        prepared: PreparedModel,
+        prepared: PreparedProject,
    ) -> ApiResult<(StoredProject, bool)> {
        let current = self.checked(id, expected)?;
-        let model = prepared.0;
-        if current.model == model {
+        if !prepared.changed(current) {
            let mut unchanged = current.clone();
-            if prepared.1.is_some() {
-                unchanged.migration = prepared.1;
+            if prepared.migration.is_some() {
+                unchanged.migration = prepared.migration;
            }
            return Ok((unchanged, false));
        }
@@ -326,8 +442,9 @@ impl StoreInner {
        let stored = StoredProject {
            resource_id: id,
            revision,
-            model,
-            migration: prepared.1,
+            model: prepared.compiled.decoded.project,
+            source_project: prepared.compiled.bundle,
+            migration: prepared.migration,
        };
        let size = self.write(&stored, false)?;
        self.adopt(stored.clone(), size);
@@ -357,17 +474,7 @@ impl StoreInner {
    }

    fn write(&self, stored: &StoredProject, create: bool) -> ApiResult<usize> {
-        // Migration observations are response metadata, never persisted facts.
-        let envelope = serde_json::json!({"resource_id": stored.resource_id, "revision": stored.revision, "model": stored.model});
-        let bytes = serde_json::to_vec(&envelope)
-            .map_err(|_| ApiError::internal("Could not serialize model."))?;
-        if bytes.len() > MAX_ENVELOPE_BYTES {
-            return Err(ApiError::new(
-                StatusCode::PAYLOAD_TOO_LARGE,
-                "model_too_large",
-                "Stored model exceeds 8 MiB.",
-            ));
-        }
+        let bytes = envelope_bytes(stored)?;
        let previous = self
            .byte_sizes
            .get(&stored.resource_id)
@@ -410,6 +517,51 @@ fn capacity_error() -> ApiError {
    )
 }

+fn present_source_project<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<SourceBundle>, D::Error> {
+    SourceBundle::deserialize(deserializer).map(Some)
+}
+
+fn envelope_bytes(stored: &StoredProject) -> ApiResult<Vec<u8>> {
+    // Migration observations are response metadata, never persisted facts.
+    #[derive(Serialize)]
+    struct Envelope<'a> {
+        resource_id: Id,
+        revision: u64,
+        model: &'a Project,
+        source_project: &'a SourceBundle,
+    }
+    let bytes = serde_json::to_vec(&Envelope {
+        resource_id: stored.resource_id,
+        revision: stored.revision,
+        model: &stored.model,
+        source_project: &stored.source_project,
+    })
+    .map_err(|_| ApiError::internal("Could not serialize accepted project."))?;
+    if bytes.len() > MAX_ENVELOPE_BYTES {
+        return Err(ApiError::new(
+            StatusCode::PAYLOAD_TOO_LARGE,
+            "project_too_large",
+            "Stored project exceeds the envelope limit.",
+        ));
+    }
+    Ok(bytes)
+}
+
+pub(crate) fn check_source_size(bundle: &SourceBundle) -> ApiResult<()> {
+    let bytes = serde_json::to_vec(bundle)
+        .map_err(|_| ApiError::bad_request("Source project cannot be serialized."))?;
+    if bytes.len() > MAX_SOURCE_PROJECT_BYTES {
+        return Err(ApiError::new(
+            StatusCode::PAYLOAD_TOO_LARGE,
+            "source_project_too_large",
+            "Serialized source project exceeds 8 MiB after identity stabilization.",
+        ));
+    }
+    Ok(())
+}
+
 pub(crate) fn check_model_size(model: &Project) -> ApiResult<()> {
    MODEL_LIMITS
        .check_model(model)
@@ -445,6 +597,46 @@ pub(crate) fn validate_model(model: &Project) -> ApiResult<()> {
 mod tests {
    use super::*;

+    #[test]
+    fn source_bytes_are_charged_on_legacy_open_update_and_delete() {
+        let directory = tempfile::tempdir().unwrap();
+        let resource_id = Id::new_v4();
+        let model = Project::empty();
+        let path = directory.path().join(format!("{resource_id}.json"));
+        let legacy_bytes = serde_json::to_vec(
+            &serde_json::json!({"resource_id":resource_id,"revision":4,"model":model}),
+        )
+        .unwrap();
+        fs::write(&path, &legacy_bytes).unwrap();
+        let store = Store::open(directory.path()).unwrap();
+        let current = store.get(resource_id).unwrap();
+        let accepted_size = envelope_bytes(&current).unwrap().len();
+        assert!(accepted_size > legacy_bytes.len());
+        assert_eq!(store.lock().unwrap().total_bytes, accepted_size);
+        assert_eq!(fs::read(&path).unwrap(), legacy_bytes);
+        let mut bundle = current.source_project.clone();
+        bundle
+            .files
+            .get_mut(&bundle.entry)
+            .unwrap()
+            .push_str(&format!("// {}\n", "rationale ".repeat(512)));
+        let candidate = PreparedProject::from_bundle(bundle).unwrap();
+        let changed = store
+            .replace(resource_id, &current.etag(), candidate)
+            .unwrap()
+            .0;
+        let changed_size = envelope_bytes(&changed).unwrap().len();
+        assert!(changed_size > accepted_size + 5_000);
+        assert_eq!(store.lock().unwrap().total_bytes, changed_size);
+        assert_eq!(fs::read(&path).unwrap().len(), changed_size);
+        drop(store);
+        let store = Store::open(directory.path()).unwrap();
+        assert_eq!(store.lock().unwrap().total_bytes, changed_size);
+        store.delete(resource_id, &changed.etag()).unwrap();
+        assert_eq!(store.lock().unwrap().total_bytes, 0);
+        assert!(store.lock().unwrap().byte_sizes.is_empty());
+    }
+
    #[test]
    fn prepared_candidates_recheck_revision_at_commit_and_failed_loser_is_not_installed() {
        let directory = tempfile::tempdir().unwrap();
@@ -456,8 +648,8 @@ mod tests {
        winner.name = "Winning review".into();
        loser.name = "Losing stale review".into();
        // Both requests completed preparation against the same original head.
-        let winner = PreparedModel::new(winner).unwrap();
-        let loser = PreparedModel::new(loser).unwrap();
+        let winner = PreparedProject::new(winner).unwrap();
+        let loser = PreparedProject::new(loser).unwrap();
        let committed = store
            .lock()
            .unwrap()