feat(analysis): stage bounded JSON codec and iterative expression ownership
This commit is contained in:
Codex
@@ -0,0 +1,427 @@
|
||||
//! Private persistence JSON boundary. Scan original bytes before Serde allocates;
|
||||
//! the scanner is a resource meter, never a second accepting JSON grammar.
|
||||
use super::FileError;
|
||||
use crate::{
|
||||
Project,
|
||||
analysis::expression_serde::{self, Scope},
|
||||
};
|
||||
use serde::{
|
||||
Deserialize,
|
||||
de::{IgnoredAny, MapAccess, Visitor},
|
||||
};
|
||||
use std::{
|
||||
fmt,
|
||||
io::{self, Read, Write},
|
||||
};
|
||||
|
||||
pub(super) const MAX_DOCUMENT_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_CONTAINERS: usize = 512;
|
||||
// This switch stays false until Windows/Linux debug/release codec + ordinary
|
||||
// lifecycle gates pass. Tests exercise the candidate through the same private
|
||||
// entry point with depth_128=true. It is not a user-controlled parser option.
|
||||
pub(super) const DEPTH_128_VALIDATED: bool = false;
|
||||
|
||||
/// Allocation cardinalities, independent of graph admission and numeric work.
|
||||
/// The default deliberately admits the existing CLI's 100000 graph objects,
|
||||
/// 250000 records and 500000 ordered references within its 64 MiB envelope.
|
||||
/// Strings have no new 1 MiB restriction: any span fitting that envelope fits.
|
||||
/// These are explicit slot/text bounds, not an assertion of process RSS.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(super) struct Profile {
|
||||
pub bytes: usize,
|
||||
pub tokens: usize,
|
||||
pub entries: usize,
|
||||
pub container_entries: usize,
|
||||
pub string_bytes: usize,
|
||||
pub total_string_bytes: usize,
|
||||
pub expression_nodes: usize,
|
||||
}
|
||||
impl Profile {
|
||||
pub const DOCUMENT: Self = Self {
|
||||
bytes: MAX_DOCUMENT_BYTES,
|
||||
tokens: 16 * 1024 * 1024,
|
||||
entries: 8 * 1024 * 1024,
|
||||
container_entries: 500_000,
|
||||
string_bytes: MAX_DOCUMENT_BYTES,
|
||||
total_string_bytes: MAX_DOCUMENT_BYTES,
|
||||
expression_nodes: expression_serde::MAX_JSON_EXPRESSION_NODES,
|
||||
};
|
||||
pub fn with_byte_limit(maximum: usize) -> Self {
|
||||
Self {
|
||||
bytes: maximum.min(MAX_DOCUMENT_BYTES),
|
||||
..Self::DOCUMENT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct Usage {
|
||||
pub tokens: usize,
|
||||
pub entries: usize,
|
||||
pub string_bytes: usize,
|
||||
pub deepest_container: usize,
|
||||
}
|
||||
#[derive(Clone, Copy)]
|
||||
struct Container {
|
||||
delimiter: u8,
|
||||
entries: usize,
|
||||
first: bool,
|
||||
}
|
||||
const EMPTY: Container = Container {
|
||||
delimiter: 0,
|
||||
entries: 0,
|
||||
first: true,
|
||||
};
|
||||
|
||||
fn limit(message: &str) -> FileError {
|
||||
FileError::Limit(message.into())
|
||||
}
|
||||
fn charge(used: &mut usize, amount: usize, maximum: usize, message: &str) -> Result<(), FileError> {
|
||||
if amount > maximum.saturating_sub(*used) {
|
||||
return Err(limit(message));
|
||||
}
|
||||
*used += amount;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn preflight(bytes: &[u8], profile: Profile) -> Result<Usage, FileError> {
|
||||
if bytes.len() > profile.bytes {
|
||||
return Err(limit("JSON input byte limit exceeded."));
|
||||
}
|
||||
// Fixed storage: no Vec/Value/String and no recursive calls during preflight.
|
||||
let mut containers = [EMPTY; MAX_CONTAINERS];
|
||||
let mut depth = 0usize;
|
||||
let mut usage = Usage::default();
|
||||
let mut cursor = 0usize;
|
||||
while cursor < bytes.len() {
|
||||
let byte = bytes[cursor];
|
||||
if byte.is_ascii_whitespace() {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
charge(
|
||||
&mut usage.tokens,
|
||||
1,
|
||||
profile.tokens,
|
||||
"JSON token limit exceeded.",
|
||||
)?;
|
||||
if !matches!(byte, b',' | b':' | b'}' | b']') && depth > 0 {
|
||||
let frame = &mut containers[depth - 1];
|
||||
if frame.first {
|
||||
charge(
|
||||
&mut frame.entries,
|
||||
1,
|
||||
profile.container_entries,
|
||||
"JSON container entry limit exceeded.",
|
||||
)?;
|
||||
charge(
|
||||
&mut usage.entries,
|
||||
1,
|
||||
profile.entries,
|
||||
"JSON aggregate entry limit exceeded.",
|
||||
)?;
|
||||
frame.first = false;
|
||||
}
|
||||
}
|
||||
match byte {
|
||||
b'{' | b'[' => {
|
||||
if depth == MAX_CONTAINERS {
|
||||
return Err(limit("JSON container depth exceeds 512."));
|
||||
}
|
||||
containers[depth] = Container {
|
||||
delimiter: byte,
|
||||
..EMPTY
|
||||
};
|
||||
depth += 1;
|
||||
usage.deepest_container = usage.deepest_container.max(depth);
|
||||
cursor += 1;
|
||||
}
|
||||
b'}' | b']' => {
|
||||
if depth == 0
|
||||
|| !matches!(
|
||||
(containers[depth - 1].delimiter, byte),
|
||||
(b'{', b'}') | (b'[', b']')
|
||||
)
|
||||
{
|
||||
return Err(FileError::Invalid(
|
||||
"Mismatched JSON container delimiter.".into(),
|
||||
));
|
||||
}
|
||||
depth -= 1;
|
||||
cursor += 1;
|
||||
}
|
||||
b',' => {
|
||||
if depth > 0 {
|
||||
containers[depth - 1].first = true;
|
||||
}
|
||||
cursor += 1;
|
||||
}
|
||||
b':' => cursor += 1,
|
||||
b'"' => {
|
||||
cursor += 1;
|
||||
let start = cursor;
|
||||
let mut closed = false;
|
||||
while cursor < bytes.len() {
|
||||
match bytes[cursor] {
|
||||
b'"' => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
b'\\' => {
|
||||
// Skip exactly one escaped byte, including an escaped
|
||||
// backslash. Serde validates actual escapes/UTF-8.
|
||||
cursor += 1;
|
||||
if cursor < bytes.len() {
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
_ => cursor += 1,
|
||||
}
|
||||
}
|
||||
let span = cursor - start;
|
||||
if span > profile.string_bytes {
|
||||
return Err(limit("JSON string byte limit exceeded."));
|
||||
}
|
||||
charge(
|
||||
&mut usage.string_bytes,
|
||||
span,
|
||||
profile.total_string_bytes,
|
||||
"JSON aggregate string byte limit exceeded.",
|
||||
)?;
|
||||
if !closed {
|
||||
return Err(FileError::Invalid("Unterminated JSON string.".into()));
|
||||
}
|
||||
cursor += 1;
|
||||
}
|
||||
_ => {
|
||||
// Count a scalar token without interpreting its value. Invalid
|
||||
// tokens and invalid whitespace must still fail the Serde pass.
|
||||
cursor += 1;
|
||||
while cursor < bytes.len()
|
||||
&& !bytes[cursor].is_ascii_whitespace()
|
||||
&& !matches!(
|
||||
bytes[cursor],
|
||||
b'{' | b'[' | b'}' | b']' | b',' | b':' | b'"'
|
||||
)
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if depth != 0 {
|
||||
return Err(FileError::Invalid("Unclosed JSON container.".into()));
|
||||
}
|
||||
Ok(usage)
|
||||
}
|
||||
|
||||
struct Version(Option<u64>);
|
||||
impl<'de> Deserialize<'de> for Version {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct Header;
|
||||
impl<'de> Visitor<'de> for Header {
|
||||
type Value = Version;
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a project header")
|
||||
}
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut access: A) -> Result<Version, A::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(field_identifier)]
|
||||
enum Key {
|
||||
#[serde(rename = "schema_version")]
|
||||
Schema,
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
let mut version = None;
|
||||
while let Some(key) = access.next_key::<Key>()? {
|
||||
match key {
|
||||
Key::Schema => {
|
||||
if version.is_some() {
|
||||
return Err(serde::de::Error::duplicate_field("schema_version"));
|
||||
}
|
||||
version = Some(access.next_value()?);
|
||||
}
|
||||
Key::Other => {
|
||||
access.next_value::<IgnoredAny>()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Version(version))
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_map(Header)
|
||||
}
|
||||
}
|
||||
|
||||
/// IgnoredAny in serde_json walks arbitrary ignored containers iteratively.
|
||||
/// This probe does not materialize ignored subtrees. Serde's scratch buffer for
|
||||
/// container markers/escaped keys is bounded by the preceding raw scan. It
|
||||
/// consumes the entire original stream and rejects duplicate/escaped versions.
|
||||
pub(super) fn version(bytes: &[u8]) -> Result<Option<u64>, FileError> {
|
||||
document_stack(|| {
|
||||
let mut deserializer = serde_json::Deserializer::from_slice(bytes);
|
||||
let version = Version::deserialize(&mut deserializer)?;
|
||||
deserializer.end()?;
|
||||
Ok(version.0)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn document_stack<R>(operation: impl FnOnce() -> R) -> R {
|
||||
// Non-Expr Project and historical schemas are finitely nested by their
|
||||
// concrete Rust types. Reserve for that outer generated Serde frame chain;
|
||||
// only recursive Expr edges use the smaller repeated boundary guard.
|
||||
stacker::maybe_grow(512 * 1024, 2 * 1024 * 1024, operation)
|
||||
}
|
||||
pub(super) fn current(
|
||||
bytes: &[u8],
|
||||
profile: Profile,
|
||||
depth_128: bool,
|
||||
) -> Result<Project, FileError> {
|
||||
preflight(bytes, profile)?;
|
||||
let _scope = Scope::decoding(profile.expression_nodes);
|
||||
document_stack(|| {
|
||||
let mut deserializer = serde_json::Deserializer::from_slice(bytes);
|
||||
if depth_128 {
|
||||
deserializer.disable_recursion_limit();
|
||||
}
|
||||
let project = Project::deserialize(&mut deserializer)?;
|
||||
deserializer.end()?;
|
||||
Ok(project)
|
||||
})
|
||||
}
|
||||
pub(super) fn legacy<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, FileError> {
|
||||
// Never disable the legacy recursion guard, including its temporary Value.
|
||||
document_stack(|| Ok(serde_json::from_slice(bytes)?))
|
||||
}
|
||||
|
||||
pub(super) fn discard_value(value: serde_json::Value) {
|
||||
let mut pending = vec![value];
|
||||
while let Some(value) = pending.pop() {
|
||||
match value {
|
||||
serde_json::Value::Array(values) => pending.extend(values),
|
||||
serde_json::Value::Object(values) => pending.extend(values.into_values()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reserve_bounded(bytes: &mut Vec<u8>, additional: usize, maximum: usize) -> io::Result<()> {
|
||||
let required = bytes.len() + additional;
|
||||
if required > bytes.capacity() {
|
||||
let next = bytes
|
||||
.capacity()
|
||||
.max(4096)
|
||||
.saturating_mul(2)
|
||||
.max(required)
|
||||
.min(maximum);
|
||||
bytes
|
||||
.try_reserve_exact(next - bytes.len())
|
||||
.map_err(io::Error::other)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct CappedWriter {
|
||||
bytes: Vec<u8>,
|
||||
maximum: usize,
|
||||
exceeded: bool,
|
||||
}
|
||||
impl CappedWriter {
|
||||
fn new(maximum: usize) -> Self {
|
||||
Self {
|
||||
bytes: Vec::new(),
|
||||
maximum,
|
||||
exceeded: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Write for CappedWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.len() > self.maximum.saturating_sub(self.bytes.len()) {
|
||||
self.exceeded = true;
|
||||
return Err(io::Error::other("JSON output byte limit exceeded."));
|
||||
}
|
||||
// Check before reserve/extend; failed output never creates a giant Vec.
|
||||
reserve_bounded(&mut self.bytes, bytes.len(), self.maximum)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compact output avoids indentation multiplying the cost of legal depth128.
|
||||
/// The returned bytes have already passed the identical reader resource profile.
|
||||
pub(super) fn encode(
|
||||
project: &Project,
|
||||
profile: Profile,
|
||||
depth_128: bool,
|
||||
) -> Result<Vec<u8>, FileError> {
|
||||
let _scope = Scope::encoding(profile.expression_nodes);
|
||||
let mut writer = CappedWriter::new(profile.bytes);
|
||||
let result = document_stack(|| serde_json::to_writer(&mut writer, project));
|
||||
if writer.exceeded {
|
||||
return Err(limit("JSON output byte limit exceeded."));
|
||||
}
|
||||
result?;
|
||||
writer.write_all(b"\n").map_err(|error| {
|
||||
if writer.exceeded {
|
||||
limit("JSON output byte limit exceeded.")
|
||||
} else {
|
||||
FileError::Io(error)
|
||||
}
|
||||
})?;
|
||||
// Establish save/reopen symmetry before filesystem mutation, including the
|
||||
// conservative production guard until the deeper profile is validated.
|
||||
let reopened = current(&writer.bytes, profile, depth_128)?;
|
||||
if reopened != *project {
|
||||
return Err(FileError::Invalid(
|
||||
"JSON serialization did not preserve the project value.".into(),
|
||||
));
|
||||
}
|
||||
Ok(writer.bytes)
|
||||
}
|
||||
|
||||
pub(super) fn read_bounded(path: &std::path::Path, maximum: usize) -> Result<Vec<u8>, FileError> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
if file.metadata()?.len() > maximum as u64 {
|
||||
return Err(limit("Input file byte limit exceeded."));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
let mut chunk = [0u8; 8192];
|
||||
loop {
|
||||
let count = file.read(&mut chunk)?;
|
||||
if count == 0 {
|
||||
return Ok(bytes);
|
||||
}
|
||||
if count > maximum.saturating_sub(bytes.len()) {
|
||||
return Err(limit("Input file byte limit exceeded."));
|
||||
}
|
||||
reserve_bounded(&mut bytes, count, maximum)?;
|
||||
bytes.extend_from_slice(&chunk[..count]);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn matches_file(path: &std::path::Path, expected: &[u8]) -> io::Result<bool> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
if file.metadata()?.len() != expected.len() as u64 {
|
||||
return Ok(false);
|
||||
}
|
||||
let mut compared = 0usize;
|
||||
let mut chunk = [0u8; 8192];
|
||||
loop {
|
||||
let count = file.read(&mut chunk)?;
|
||||
if count == 0 {
|
||||
return Ok(compared == expected.len());
|
||||
}
|
||||
if expected.get(compared..compared + count) != Some(&chunk[..count]) {
|
||||
return Ok(false);
|
||||
}
|
||||
compared += count;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,612 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
Element, ElementBody, Id, OaBody,
|
||||
analysis::{
|
||||
AnalysisType, Binding, CalculationDefinition, Definition, DefinitionBody, Expr, Parameter,
|
||||
ParameterId, ScalarType, UnaryOp,
|
||||
},
|
||||
language::ParseLimits,
|
||||
persistence,
|
||||
};
|
||||
use std::{
|
||||
fmt::Write as _,
|
||||
process::{Command, Stdio},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
const IDENTITY: &str = "00000000-0000-0000-0000-000000000001";
|
||||
const PARAMETER: &str = "00000000-0000-0000-0000-000000000002";
|
||||
const MAIN: &str = "00000000-0000-0000-0000-000000000003";
|
||||
fn id(value: &str) -> Id {
|
||||
Id::parse_str(value).unwrap()
|
||||
}
|
||||
fn boolean_type() -> AnalysisType {
|
||||
AnalysisType::Scalar(ScalarType::Boolean)
|
||||
}
|
||||
fn definition(id: Id, alias: &str, parameters: Vec<Parameter>, body: Expr) -> Definition {
|
||||
Definition {
|
||||
id,
|
||||
alias: alias.into(),
|
||||
name: alias.into(),
|
||||
description: String::new(),
|
||||
body: DefinitionBody::Calculation(CalculationDefinition {
|
||||
parameters,
|
||||
result: boolean_type(),
|
||||
body: Binding::Expression(body),
|
||||
}),
|
||||
}
|
||||
}
|
||||
fn project() -> Project {
|
||||
let mut project = Project::empty();
|
||||
project.id = Id::from_u128(1000);
|
||||
project.definitions.insert(
|
||||
id(IDENTITY),
|
||||
definition(
|
||||
id(IDENTITY),
|
||||
"identity",
|
||||
vec![Parameter {
|
||||
id: ParameterId(id(PARAMETER)),
|
||||
name: "value".into(),
|
||||
value_type: boolean_type(),
|
||||
}],
|
||||
Expr::Parameter(ParameterId(id(PARAMETER))),
|
||||
),
|
||||
);
|
||||
project.definitions.insert(
|
||||
id(MAIN),
|
||||
definition(id(MAIN), "main", vec![], Expr::Boolean(true)),
|
||||
);
|
||||
project
|
||||
}
|
||||
fn wire(expression: &str) -> Vec<u8> {
|
||||
let shallow = serde_json::to_string(&project()).unwrap();
|
||||
assert_eq!(shallow.matches(r#"{"Boolean":true}"#).count(), 1);
|
||||
shallow
|
||||
.replace(r#"{"Boolean":true}"#, expression)
|
||||
.into_bytes()
|
||||
}
|
||||
fn chain(depth: usize, call: bool) -> String {
|
||||
assert!(depth > 0);
|
||||
let prefix = if call {
|
||||
format!(r#"{{"Call":{{"definition":{{"Local":"{IDENTITY}"}},"arguments":{{"{PARAMETER}":"#)
|
||||
} else {
|
||||
r#"{"Unary":{"op":"Not","operand":"#.into()
|
||||
};
|
||||
let suffix = if call { "}}}" } else { "}}" };
|
||||
format!(
|
||||
"{}{}{}",
|
||||
prefix.repeat(depth - 1),
|
||||
r#"{"Boolean":true}"#,
|
||||
suffix.repeat(depth - 1)
|
||||
)
|
||||
}
|
||||
fn candidate(bytes: &[u8], profile: Profile) -> Result<Project, FileError> {
|
||||
persistence::decode_with_report_profile(bytes, ParseLimits::UNLIMITED, profile, true)
|
||||
.map(|decoded| decoded.project)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resource_preflight_has_exact_lowerable_boundaries() {
|
||||
let bytes = br#"{"key":["a\\\"[{}]",0,true],"other":"\u0061"}"#;
|
||||
let usage = preflight(bytes, Profile::DOCUMENT).unwrap();
|
||||
let exact = Profile {
|
||||
bytes: bytes.len(),
|
||||
tokens: usage.tokens,
|
||||
entries: usage.entries,
|
||||
container_entries: 3,
|
||||
string_bytes: 9,
|
||||
total_string_bytes: usage.string_bytes,
|
||||
..Profile::DOCUMENT
|
||||
};
|
||||
preflight(bytes, exact).unwrap();
|
||||
for lower in [
|
||||
Profile {
|
||||
bytes: exact.bytes - 1,
|
||||
..exact
|
||||
},
|
||||
Profile {
|
||||
tokens: exact.tokens - 1,
|
||||
..exact
|
||||
},
|
||||
Profile {
|
||||
entries: exact.entries - 1,
|
||||
..exact
|
||||
},
|
||||
Profile {
|
||||
container_entries: 2,
|
||||
..exact
|
||||
},
|
||||
Profile {
|
||||
total_string_bytes: exact.total_string_bytes - 1,
|
||||
..exact
|
||||
},
|
||||
Profile {
|
||||
string_bytes: exact.string_bytes - 1,
|
||||
..exact
|
||||
},
|
||||
] {
|
||||
assert!(matches!(preflight(bytes, lower), Err(FileError::Limit(_))));
|
||||
}
|
||||
let deepest = format!("{}0{}", "[".repeat(512), "]".repeat(512));
|
||||
assert_eq!(
|
||||
preflight(deepest.as_bytes(), Profile::DOCUMENT)
|
||||
.unwrap()
|
||||
.deepest_container,
|
||||
512
|
||||
);
|
||||
assert!(matches!(
|
||||
preflight(format!("[{deepest}]").as_bytes(), Profile::DOCUMENT),
|
||||
Err(FileError::Limit(_))
|
||||
));
|
||||
// A huge ignored scalar array exhausts preflight before a header is decoded.
|
||||
assert!(matches!(
|
||||
preflight(
|
||||
br#"{"schema_version":8,"junk":[0,0,0,0]}"#,
|
||||
Profile {
|
||||
container_entries: 3,
|
||||
..Profile::DOCUMENT
|
||||
}
|
||||
),
|
||||
Err(FileError::Limit(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_is_not_a_parallel_json_grammar() {
|
||||
for bytes in [
|
||||
br#"{"schema_version":8}{}"#.as_slice(),
|
||||
br#"{"schema_version":8,}"#,
|
||||
br#"{"schema_version":8 "junk":0}"#,
|
||||
br#"{"schema_version":8,"schema_version":8}"#,
|
||||
br#"{"schema_version":8,"\u0073chema_version":8}"#,
|
||||
br#"{"schema_version":8,"junk":"\q"}"#,
|
||||
br#"{"schema_version":8,"junk":"unfinished}"#,
|
||||
br#"{"schema_version":8,"junk":[}"#,
|
||||
] {
|
||||
assert!(candidate(bytes, Profile::DOCUMENT).is_err(), "{bytes:?}");
|
||||
}
|
||||
let mut invalid_utf8 = wire(r#"{"Boolean":true}"#);
|
||||
let offset = invalid_utf8
|
||||
.windows(7)
|
||||
.position(|window| window == b"mission")
|
||||
.unwrap();
|
||||
invalid_utf8[offset] = 0xff;
|
||||
assert!(candidate(&invalid_utf8, Profile::DOCUMENT).is_err());
|
||||
let escaped = String::from_utf8(wire(r#"{"Boolean":true}"#))
|
||||
.unwrap()
|
||||
.replace(r#""schema_version""#, r#""\u0073chema_version""#);
|
||||
assert_eq!(
|
||||
candidate(escaped.as_bytes(), Profile::DOCUMENT).unwrap(),
|
||||
project()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_serde_keeps_each_existing_expression_variant() {
|
||||
let n = r#"{"Number":{"numerator":"1","denominator":"1"}}"#;
|
||||
let b = r#"{"Boolean":true}"#;
|
||||
let expressions = [
|
||||
b.to_string(),
|
||||
n.to_string(),
|
||||
format!(
|
||||
r#"{{"Quantity":{{"magnitude":{{"numerator":"1","denominator":"1"}},"unit":{{"Local":"{IDENTITY}"}}}}}}"#
|
||||
),
|
||||
format!(r#"{{"Property":"{IDENTITY}"}}"#),
|
||||
format!(r#"{{"Parameter":"{PARAMETER}"}}"#),
|
||||
format!(r#"{{"List":[{n},{n}]}}"#),
|
||||
format!(r#"{{"Unary":{{"op":"Not","operand":{b}}}}}"#),
|
||||
format!(r#"{{"Binary":{{"op":"Add","left":{n},"right":{n}}}}}"#),
|
||||
format!(r#"{{"If":{{"condition":{b},"when_true":{n},"when_false":{n}}}}}"#),
|
||||
format!(
|
||||
r#"{{"Call":{{"definition":{{"Local":"{IDENTITY}"}},"arguments":{{"{PARAMETER}":{b}}}}}}}"#
|
||||
),
|
||||
format!(r#"{{"Sum":{{"values":{{"List":[{n}]}},"initial":{n}}}}}"#),
|
||||
format!(r#"{{"Power":{{"base":{n},"exponent":2}}}}"#),
|
||||
format!(r#"{{"AsKind":{{"kind":{{"Local":"{IDENTITY}"}},"value":{n}}}}}"#),
|
||||
];
|
||||
for original in expressions {
|
||||
let value: Expr = serde_json::from_str(&original).unwrap();
|
||||
let copy = value.clone();
|
||||
assert!(value == copy);
|
||||
assert_eq!(serde_json::to_string(&value).unwrap(), original);
|
||||
assert_eq!(format!("{value:?}"), format!("{copy:?}"));
|
||||
}
|
||||
let list = Expr::List(vec![Expr::Boolean(true), Expr::Boolean(false)]);
|
||||
assert_eq!(format!("{list:?}"), "List([Boolean(true), Boolean(false)])");
|
||||
assert!(list != Expr::List(vec![Expr::Boolean(false), Expr::Boolean(true)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_profile_does_not_contract_declared_cli_collection_counts() {
|
||||
// Exercise original-byte typed construction at the existing graph ceiling.
|
||||
// This gate checks codec/count compatibility, not 100000-object UI speed.
|
||||
let mut model = Project::empty();
|
||||
model.id = Id::from_u128(1_000_000);
|
||||
for value in 1..=100_000 {
|
||||
let id = Id::from_u128(value);
|
||||
model.elements.insert(
|
||||
id,
|
||||
Element {
|
||||
id,
|
||||
alias: format!("activity_{value}"),
|
||||
name: String::new(),
|
||||
description: String::new(),
|
||||
parent: None,
|
||||
body: ElementBody::Oa(OaBody::Activity),
|
||||
},
|
||||
);
|
||||
}
|
||||
let bytes = encode(&model, Profile::DOCUMENT, true).unwrap();
|
||||
let decoded = current(&bytes, Profile::DOCUMENT, true).unwrap();
|
||||
assert_eq!(decoded.elements.len(), 100_000);
|
||||
let cli = ParseLimits {
|
||||
max_source_bytes: 64 * 1024 * 1024,
|
||||
max_graph_objects: 100_000,
|
||||
max_records: 250_000,
|
||||
max_step_references: 500_000,
|
||||
};
|
||||
cli.check_model(&decoded).unwrap();
|
||||
let references = format!(
|
||||
"[{}\"{IDENTITY}\"]",
|
||||
format!("\"{IDENTITY}\",").repeat(499_999)
|
||||
);
|
||||
preflight(references.as_bytes(), Profile::DOCUMENT).unwrap();
|
||||
assert!(matches!(
|
||||
preflight(
|
||||
format!("[\"{IDENTITY}\",{}]", &references[1..references.len() - 1]).as_bytes(),
|
||||
Profile::DOCUMENT
|
||||
),
|
||||
Err(FileError::Limit(_))
|
||||
));
|
||||
// Metadata over the initially proposed 1 MiB limit remains admissible.
|
||||
let mut text = Project::empty();
|
||||
text.description = "x".repeat(1024 * 1024 + 1);
|
||||
let bytes = encode(&text, Profile::DOCUMENT, true).unwrap();
|
||||
assert_eq!(current(&bytes, Profile::DOCUMENT, true).unwrap(), text);
|
||||
eprintln!(
|
||||
"JSON profile: graph bytes={}, ordered-reference bytes={}, max container entries={}",
|
||||
serde_json::to_string(&model).unwrap().len(),
|
||||
references.len(),
|
||||
Profile::DOCUMENT.container_entries
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_save_always_reopens_with_its_reader_profile() {
|
||||
let deep = candidate(&wire(&chain(128, true)), Profile::DOCUMENT).unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("project.json");
|
||||
std::fs::write(&path, b"previous").unwrap();
|
||||
let saved = persistence::save_checked(&path, &deep, Some(b"previous"));
|
||||
if DEPTH_128_VALIDATED {
|
||||
let bytes = saved.unwrap();
|
||||
assert_eq!(persistence::decode(&bytes).unwrap(), deep);
|
||||
} else {
|
||||
assert!(saved.is_err());
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"previous");
|
||||
}
|
||||
}
|
||||
|
||||
fn accepted_phase() {
|
||||
for call in [false, true] {
|
||||
let bytes = wire(&chain(128, call));
|
||||
let usage = preflight(&bytes, Profile::DOCUMENT).unwrap();
|
||||
let model = candidate(&bytes, Profile::DOCUMENT).unwrap();
|
||||
let copy = model.clone();
|
||||
assert!(model == copy);
|
||||
let debug = format!("{model:?}");
|
||||
assert!(!debug.is_empty());
|
||||
let output = encode(&model, Profile::DOCUMENT, true).unwrap();
|
||||
assert!(candidate(&output, Profile::DOCUMENT).unwrap() == model);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("deep.json");
|
||||
let saved = persistence::save_checked_profile(&path, &model, None, Profile::DOCUMENT, true)
|
||||
.unwrap();
|
||||
assert!(
|
||||
candidate(
|
||||
&read_bounded(&path, MAX_DOCUMENT_BYTES).unwrap(),
|
||||
Profile::DOCUMENT
|
||||
)
|
||||
.unwrap()
|
||||
== model
|
||||
);
|
||||
assert_eq!(saved, output);
|
||||
eprintln!(
|
||||
"accepted call={call} depth128 input={} output={} tokens={} entries={} raw strings={} containers={} debug={}",
|
||||
bytes.len(),
|
||||
output.len(),
|
||||
usage.tokens,
|
||||
usage.entries,
|
||||
usage.string_bytes,
|
||||
usage.deepest_container,
|
||||
debug.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
fn refused_phase() {
|
||||
for call in [false, true] {
|
||||
let error = candidate(&wire(&chain(129, call)), Profile::DOCUMENT).unwrap_err();
|
||||
assert!(error.to_string().contains("logical depth"), "{error}");
|
||||
}
|
||||
let wide = format!(
|
||||
r#"{{"List":[{}null]}}"#,
|
||||
r#"{"Boolean":true},"#.repeat(1024)
|
||||
);
|
||||
let error = candidate(&wire(&wide), Profile::DOCUMENT).unwrap_err();
|
||||
assert!(error.to_string().contains("1024"), "{error}");
|
||||
let bytes = wire(&chain(128, true));
|
||||
let exact_model = candidate(
|
||||
&bytes,
|
||||
Profile {
|
||||
expression_nodes: 129,
|
||||
..Profile::DOCUMENT
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let error = candidate(
|
||||
&bytes,
|
||||
Profile {
|
||||
expression_nodes: 128,
|
||||
..Profile::DOCUMENT
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
// Identity's Parameter plus main's 128-node expression exceed this total.
|
||||
assert!(error.to_string().contains("allocation count"), "{error}");
|
||||
let scan = preflight(&bytes, Profile::DOCUMENT).unwrap();
|
||||
assert!(matches!(
|
||||
candidate(
|
||||
&bytes,
|
||||
Profile {
|
||||
tokens: scan.tokens - 1,
|
||||
..Profile::DOCUMENT
|
||||
}
|
||||
),
|
||||
Err(FileError::Limit(_))
|
||||
));
|
||||
document_stack(|| {
|
||||
use crate::analysis::{AdmissionLimits, admit};
|
||||
let events = admit(&exact_model, AdmissionLimits::REFERENCE)
|
||||
.unwrap()
|
||||
.admission_usage()
|
||||
.static_events;
|
||||
admit(
|
||||
&exact_model,
|
||||
AdmissionLimits {
|
||||
max_static_events: events,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let error = match admit(
|
||||
&exact_model,
|
||||
AdmissionLimits {
|
||||
max_static_events: events - 1,
|
||||
},
|
||||
) {
|
||||
Ok(_) => panic!("Static work below the measured exact count must refuse."),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code, "analysis_limit");
|
||||
});
|
||||
let mut tree = r#"{"Boolean":true}"#.to_owned();
|
||||
for _ in 0..15 {
|
||||
tree = format!(r#"{{"Binary":{{"op":"And","left":{tree},"right":{tree}}}}}"#);
|
||||
}
|
||||
// 65535 balanced nodes plus one unary node is the stored-tree boundary.
|
||||
tree = format!(r#"{{"Unary":{{"op":"Not","operand":{tree}}}}}"#);
|
||||
let at_tree_limit = wire(&tree);
|
||||
current(&at_tree_limit, Profile::DOCUMENT, true).unwrap();
|
||||
let beyond = wire(&format!(r#"{{"Unary":{{"op":"Not","operand":{tree}}}}}"#));
|
||||
assert!(
|
||||
current(&beyond, Profile::DOCUMENT, true)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("65536")
|
||||
);
|
||||
// No downstream work after refusal poisons the next ordinary operation.
|
||||
candidate(&wire(&chain(128, true)), Profile::DOCUMENT).unwrap();
|
||||
}
|
||||
fn malformed_phase() {
|
||||
let deep = chain(127, false);
|
||||
for (label, expression) in [
|
||||
(
|
||||
"duplicate operand",
|
||||
format!(r#"{{"Unary":{{"op":"Not","operand":{deep},"operand":null}}}}"#),
|
||||
),
|
||||
(
|
||||
"escaped duplicate",
|
||||
format!(r#"{{"Unary":{{"op":"Not","operand":{deep},"\u006fperand":null}}}}"#),
|
||||
),
|
||||
(
|
||||
"unknown field",
|
||||
format!(r#"{{"Unary":{{"op":"Not","operand":{deep},"surprise":null}}}}"#),
|
||||
),
|
||||
(
|
||||
"duplicate enum",
|
||||
format!(r#"{{"Unary":{{"op":"Not","operand":{deep}}},"Boolean":true}}"#),
|
||||
),
|
||||
(
|
||||
"duplicate parameter",
|
||||
format!(
|
||||
r#"{{"Call":{{"definition":{{"Local":"{IDENTITY}"}},"arguments":{{"{PARAMETER}":{deep},"\u0030{}":null}}}}}}"#,
|
||||
&PARAMETER[1..]
|
||||
),
|
||||
),
|
||||
] {
|
||||
let error = candidate(&wire(&expression), Profile::DOCUMENT).unwrap_err();
|
||||
assert!(
|
||||
!error.to_string().contains("logical depth"),
|
||||
"{label}: {error}"
|
||||
);
|
||||
eprintln!("{label}: {error}");
|
||||
}
|
||||
let complete = String::from_utf8(wire(&chain(128, true))).unwrap();
|
||||
let duplicate_late = format!(
|
||||
r#"{},"name":"late duplicate"}}"#,
|
||||
&complete[..complete.len() - 1]
|
||||
);
|
||||
assert!(candidate(duplicate_late.as_bytes(), Profile::DOCUMENT).is_err());
|
||||
assert!(candidate(format!("{complete} null").as_bytes(), Profile::DOCUMENT).is_err());
|
||||
let invalid_surrogate = complete.replace(r#""description":"""#, r#""description":"\uD800""#);
|
||||
assert!(candidate(invalid_surrogate.as_bytes(), Profile::DOCUMENT).is_err());
|
||||
// Fully rejected raw depth also tears down without a recursive JSON Value.
|
||||
let excessive = format!("{}0{}", "[".repeat(100_000), "]".repeat(100_000));
|
||||
assert!(matches!(
|
||||
preflight(excessive.as_bytes(), Profile::DOCUMENT),
|
||||
Err(FileError::Limit(_))
|
||||
));
|
||||
}
|
||||
fn ownership_phase() {
|
||||
let mut value = Expr::Boolean(true);
|
||||
for _ in 1..100_000 {
|
||||
value = Expr::Unary {
|
||||
op: UnaryOp::Not,
|
||||
operand: Box::new(value),
|
||||
};
|
||||
}
|
||||
let mut copy = value.clone();
|
||||
assert!(value == copy);
|
||||
let mut cursor = &mut copy;
|
||||
loop {
|
||||
match cursor {
|
||||
Expr::Unary { operand, .. } => cursor = operand,
|
||||
Expr::Boolean(leaf) => {
|
||||
*leaf = false;
|
||||
break;
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
assert!(value != copy);
|
||||
// A streaming formatter avoids retaining a 100000-node Debug string.
|
||||
struct Counter(usize);
|
||||
impl fmt::Write for Counter {
|
||||
fn write_str(&mut self, value: &str) -> fmt::Result {
|
||||
self.0 += value.len();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut debug = Counter(0);
|
||||
write!(&mut debug, "{value:?}").unwrap();
|
||||
assert!(debug.0 > 100_000);
|
||||
assert!(serde_json::to_writer(io::sink(), &value).is_err());
|
||||
drop(value);
|
||||
drop(copy);
|
||||
eprintln!(
|
||||
"iterative manual100000 ownership/debug/drop completed; debug bytes={}",
|
||||
debug.0
|
||||
);
|
||||
}
|
||||
fn atomic_phase() {
|
||||
let model = project();
|
||||
let bytes = encode(&model, Profile::DOCUMENT, true).unwrap();
|
||||
let exact = Profile {
|
||||
bytes: bytes.len(),
|
||||
..Profile::DOCUMENT
|
||||
};
|
||||
assert_eq!(encode(&model, exact, true).unwrap(), bytes);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("preserved.json");
|
||||
std::fs::write(&path, b"original").unwrap();
|
||||
let error = persistence::save_checked_profile(
|
||||
&path,
|
||||
&model,
|
||||
Some(b"original"),
|
||||
Profile {
|
||||
bytes: bytes.len() - 1,
|
||||
..Profile::DOCUMENT
|
||||
},
|
||||
true,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, FileError::Limit(_)));
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"original");
|
||||
assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1);
|
||||
let error = persistence::save_checked_profile(
|
||||
&path,
|
||||
&model,
|
||||
Some(b"different"),
|
||||
Profile::DOCUMENT,
|
||||
true,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, FileError::Conflict));
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"original");
|
||||
assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1);
|
||||
assert!(read_bounded(&path, 7).is_err());
|
||||
assert_eq!(read_bounded(&path, 8).unwrap(), b"original");
|
||||
let mut writer = CappedWriter::new(3);
|
||||
writer.write_all(b"abc").unwrap();
|
||||
let capacity = writer.bytes.capacity();
|
||||
assert!(writer.write_all(b"d").is_err());
|
||||
assert_eq!(writer.bytes, b"abc");
|
||||
assert_eq!(writer.bytes.capacity(), capacity);
|
||||
}
|
||||
|
||||
/// Root runs this exact test in each approved target/profile. Each phase runs in
|
||||
/// another process so a stack overflow is an explicit failed phase, and inside
|
||||
/// a 128 KiB thread (smaller than the ordinary test stack). No RUST_MIN_STACK
|
||||
/// override, ignored test, release-only path or increased test stack.
|
||||
#[test]
|
||||
fn codec_lifecycle_subprocess() {
|
||||
const PHASE: &str = "ARCADIA_CODEC_LIFECYCLE_PHASE";
|
||||
if let Ok(phase) = std::env::var(PHASE) {
|
||||
eprintln!("codec child phase={phase}, configured thread stack=131072");
|
||||
std::thread::Builder::new()
|
||||
.name(format!("codec-{phase}"))
|
||||
.stack_size(128 * 1024)
|
||||
.spawn(move || match phase.as_str() {
|
||||
"accepted" => accepted_phase(),
|
||||
"refused" => refused_phase(),
|
||||
"malformed" => malformed_phase(),
|
||||
"ownership" => ownership_phase(),
|
||||
"atomic" => atomic_phase(),
|
||||
_ => panic!("Unknown codec phase"),
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
return;
|
||||
}
|
||||
for phase in ["accepted", "refused", "malformed", "ownership", "atomic"] {
|
||||
let mut child = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"persistence::json_codec::tests::codec_lifecycle_subprocess",
|
||||
"--test-threads=1",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(PHASE, phase)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
if child.try_wait().unwrap().is_some() {
|
||||
break;
|
||||
}
|
||||
if start.elapsed() > Duration::from_secs(120) {
|
||||
child.kill().unwrap();
|
||||
let output = child.wait_with_output().unwrap();
|
||||
panic!(
|
||||
"codec {phase} timed out: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
let output = child.wait_with_output().unwrap();
|
||||
eprintln!(
|
||||
"codec {phase}: status={} elapsed={:?}\n{}{}",
|
||||
output.status,
|
||||
start.elapsed(),
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"codec phase {phase} aborted or failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user