Files
ARCADIA-Studio/harness/experiments/analysis-mixed-project-20260924/verify.py
T

360 lines
23 KiB
Python

"""Read-only Project receipt/Git consistency verifier; never runs archived code.
Reuses the published mixed-syntax verifier's ZIP/manifest, unique JSON, immutable
Git input, actual listing/outcome and exact-command design. Proposal: not run yet.
"""
from pathlib import Path, PurePosixPath, PureWindowsPath
from datetime import datetime
import argparse
import hashlib
import io
import json
import re
import subprocess
import zipfile
FAILED = 'b9a31bb848a4f74f097aa8a09351cc82c98b5815'
TESTED = '71cadfa0cd0e112caa29588d2494753301f3cb6c'
SOURCE = 'aaec78fcc9b273d1485497126392afdd1fb2e54e'
BASELINE = 'a578ac98a06772a60bec00ae287e65650680d119'
PUBLISHED = '85cc27fed02ce770c30c9447f35c4ab4d24f9644'
BASELINE_PATH = 'harness/experiments/analysis-mixed-syntax-20260923/evidence.zip'
BASELINE_HASH = 'f39aab1639ce14e4fe61744cf8881b0883ecbe131fefd6bc65c18edd8c81215d'
SCOPE = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'build',
'crates', 'assets', 'docs', 'examples', 'harness/fixtures', 'vendor']
PREFIX = ['cargo', 'test', '--locked', '--offline', '-p', 'oa-core', '--lib']
PROJECT = 'language::analysis_source::mixed::project::tests::'
ADMISSION = 'validation::shared_admission_tests::'
PROJECT_NAMES = [PROJECT + name for name in (
'catalog_alias_first_entry_first_and_canonical_occurrence_owners',
'complete_global_arena_scoped_calls_and_coincident_file_offsets',
'complete_oa_sa_forward_model_warning_and_original_authority',
'complete_static_architecture_and_unicode_warning_mapping',
'incomplete_call_borrows_large_signature_and_refuses_at_single_admission',
'inherited_storage_failure_and_success_drop_deep_syntax_on_128k_stack',
'mapped_warning_path_copies_are_admitted_before_retention',
'ordinary_steps_add_once_to_inherited_typed_reference_budget',
'public_source3_and_source7_preserve_legacy_scratch_defaults_and_errors',
'retained_nested_record_policy_cannot_be_replaced_at_resolution',
'scoped_parameter_faults_keep_real_file_and_unknown_fields_use_owner_fallback',
'selector_counts_branch_and_context_chains_but_excludes_historical_and_scalars',
'simultaneous_catalog_storage_is_not_individually_reset')]
ADMISSION_NAMES = [ADMISSION + name for name in (
'static_failure_returns_no_metadata_and_skips_architectural_warnings',
'successful_static_metadata_survives_architectural_warnings')]
FAILED_NAMES = {PROJECT_NAMES[2], PROJECT_NAMES[3]}
INTEGRATIONS = {
'route-defaults': ('route_source', 'omitted_schema_three_routes_default_to_empty_and_v2_reader_stays_frozen'),
'route-errors': ('route_source', 'invalid_route_source_has_located_errors_and_no_silent_nested_defaults'),
'fragment-coordinates': ('source_project', 'source_errors_keep_exact_fragment_unicode_positions'),
'aggregate-references': ('source_project', 'duplicate_include_edges_count_toward_admission_and_ordered_references_are_aggregate'),
}
HELPERS = {
FAILED: 'bb3e467ba12d55affbb8802987dfcdeb45c62aff507e16d21f5a106e8b0651a4',
TESTED: 'a3b965c6d4d9f3f7d3d55ea0f142d16e43fccd9adcd9b8121615a2ea16df8af8',
SOURCE: '46c9ee5f4130e7854855e44b1509ecc0833dd4a3c41a1f3251955b0ac79cc154',
}
GATES = {}
for short, source, kinds in (
('b9a31bb', FAILED, ('clippy', 'list', 'focused')),
('71cadfa', TESTED, ('clippy', 'list', 'focused', 'admission', 'full', *INTEGRATIONS, 'fmt')),
('aaec78f', SOURCE, ('fmt',)),
):
for kind in kinds:
GATES[f'project-{kind}-{short}-1'] = (source, kind)
NOTES = {
'notes/private-handoff.md', 'notes/tested-readback.json', 'notes/successor-format-readback.json',
'diffs/b9a31bb-to-71cadfa.patch', 'diffs/71cadfa-to-aaec78f.patch', 'baseline-reference.json',
}
ROOT = 'C:/Users/User/Documents/Git/ARCADIA-Studio'
def require(condition, message):
if not condition:
raise ValueError(message)
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def unique_object(pairs):
result = {}
for key, value in pairs:
require(key not in result, 'Duplicate JSON key: ' + key)
result[key] = value
return result
def document(raw):
return json.loads(raw, object_pairs_hook=unique_object)
def safe_name(name):
require(isinstance(name, str) and name and '\\' not in name and ':' not in name,
'Unsafe archive name')
p = PurePosixPath(name)
require(not p.is_absolute() and str(p) == name and '..' not in p.parts,
'Unsafe archive path: ' + name)
return name
def git(repo, *arguments, data=None):
# Ignore replace objects and user-configured diff drivers/rendering. These
# fixed options reproduce the archived conventional Git patch format.
options = []
if arguments[0] == 'diff':
options = ['--no-ext-diff', '--no-textconv', '--no-color',
'--src-prefix=a/', '--dst-prefix=b/', '--line-prefix=',
'--diff-algorithm=myers', '--indent-heuristic',
'--unified=3', '--abbrev=7', '--no-renames']
return subprocess.check_output(
['git', '--no-replace-objects', '-c', 'color.ui=false', '-c', 'core.quotePath=true',
'-C', str(repo), arguments[0], *options, *arguments[1:]], input=data)
def git_inputs(repo, source, current):
require(current['source'] == source, 'Input source mismatch')
records = current['inputs']
require(len(records) == 544, 'Unexpected input count')
paths = git(repo, 'ls-tree', '-r', '--name-only', source, *SCOPE).decode().splitlines()
require(paths == [item['path'] for item in records], 'Git input membership/order mismatch')
raw = git(repo, 'cat-file', '--batch', data=''.join(f'{source}:{p}\n' for p in paths).encode())
offset = 0
for item in records:
end = raw.index(b'\n', offset)
_, kind, size = raw[offset:end].split()
size = int(size)
blob = raw[end + 1:end + 1 + size]
require(kind == b'blob' and size == item['bytes'] and sha(blob) == item['sha256'],
'Git input bytes mismatch: ' + item['path'])
offset = end + size + 2
require(offset == len(raw), 'Trailing Git batch bytes')
def listing(log):
names = re.findall(r'^([^\r\n]+): test\r?$', log, re.M)
require(len(names) == len(set(names)), 'Duplicate listed test')
require(re.findall(r'^(\d+) tests, (\d+) benchmarks\r?$', log, re.M) == [(str(len(names)), '0')],
'Listing footer mismatch')
return names
def outcomes(log, expected, passed, failed, filtered):
rows = re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', log, re.M)
require(len(rows) == len({name for name, _ in rows}), 'Duplicate runtime outcome')
actual = dict(rows)
require(set(actual) == set(expected), 'Missing/extra runtime outcomes')
require(sum(v == 'ok' for v in actual.values()) == passed and
sum(v == 'FAILED' for v in actual.values()) == failed and
'ignored' not in actual.values(), 'Runtime status mismatch')
summaries = re.findall(r'^test result: (ok|FAILED)\. (\d+) passed; (\d+) failed; '
r'(\d+) ignored; (\d+) measured; (\d+) filtered out;', log, re.M)
require(summaries == [('FAILED' if failed else 'ok', str(passed), str(failed), '0', '0', str(filtered))],
'Runtime footer mismatch')
return actual
def failure_blocks(log, names):
blocks = re.findall(r'^---- ([^\r\n]+) stdout ----\r?\n(.*?)(?=^---- |^failures:\r?$|\Z)', log, re.M | re.S)
require(len(blocks) == len(names) and {name for name, _ in blocks} == set(names),
'Failure-block membership mismatch')
tails = re.findall(r'^failures:\r?\n((?: [^\r\n]+\r?\n)+)', log, re.M)
require(len(tails) == 1 and tails[0].split() == sorted(names), 'Failure footer changed')
return dict(blocks)
def causes(log, names):
result = {}
for name, block in failure_blocks(log, names).items():
details = [line for line in block.splitlines() if line.startswith('called `Result::unwrap()`')]
require(len(details) == 1 and 'Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.' in details[0]
and 'invalid_source' in details[0], 'Failure cause changed: ' + name)
result[name] = details[0]
return result
def format_blocks(log):
parts = re.split(r'^Diff in [^\n]*?[/\\]as3batch[/\\]([^\n]+):(\d+):\n', log.replace('\r\n', '\n'), flags=re.M)
require(not parts[0] and len(parts) > 1 and (len(parts) - 1) % 3 == 0, 'Unexpected formatting output')
result = {}
for i in range(1, len(parts), 3):
key = (parts[i].replace('\\', '/'), int(parts[i + 1]))
require(key not in result, 'Duplicate formatter block')
result[key] = parts[i + 2]
return result
def norm(path):
return PureWindowsPath(path).as_posix().lower()
def baseline(repo, reference):
expected = {'commit': PUBLISHED, 'path': BASELINE_PATH, 'sha256': BASELINE_HASH, 'source': BASELINE}
require(reference == expected, 'Published baseline reference changed')
raw = git(repo, 'show', PUBLISHED + ':' + BASELINE_PATH)
require(sha(raw) == BASELINE_HASH, 'Published baseline archive differs')
with zipfile.ZipFile(io.BytesIO(raw)) as z:
require(len(z.namelist()) == len(set(z.namelist())), 'Duplicate published baseline member')
listed = z.read('gates/mixed-direct-list-a578-1.log').decode()
full = z.read('gates/mixed-direct-full-a578-1.log').decode()
fmt = z.read('gates/mixed-fmt-a578-1.log').decode()
historical = z.read('baseline/full-f1d79df.log').decode()
names = listing(listed)
require(len(names) == 194, 'Published baseline name count differs')
statuses = outcomes(full, names, 178, 16, 0)
bad = {name for name, status in statuses.items() if status == 'FAILED'}
old = dict(re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', historical, re.M))
require(len(old) == 152 and all(statuses.get(name) == status for name, status in old.items()),
'Published historical status link differs')
return names, statuses, causes(full, bad), format_blocks(fmt)
def verify_payload(payload, manifest, repo):
require(manifest['schema'] == 1 and manifest['source'] == SOURCE and manifest['runtime_source'] == TESTED,
'Manifest source/schema mismatch')
expected_members = NOTES | {'gates/launchers/' + value + '.py' for value in HELPERS.values()}
expected_members |= {'gates/' + stem + suffix for stem in GATES for suffix in ('.result.json', '.log', '.inputs.json')}
require(set(payload) == set(manifest['payloads']) == expected_members, 'Missing/extra package payload')
for name, raw in payload.items():
safe_name(name)
require(manifest['payloads'][name] == {'bytes': len(raw), 'sha256': sha(raw)}, 'Payload hash/size mismatch: ' + name)
old_names, old_status, old_causes, old_fmt = baseline(repo, document(payload['baseline-reference.json']))
records, logs, inventories = {}, {}, {}
for stem, (source, kind) in GATES.items():
root = 'gates/' + stem
r = document(payload[root + '.result.json'])
log = payload[root + '.log'].decode('utf-8')
require(r['source'] == r['head_after'] == source and r['changed_inputs'] == [] and r['verification_failures'] == [], 'Gate source/verification drift: ' + stem)
require(datetime.fromisoformat(r['start']) < datetime.fromisoformat(r['end']) and r['elapsed_monotonic_seconds'] > 0, 'Gate not closed')
env = r['environment']
require(env['CARGO_BUILD_JOBS'] == '1' and env['CARGO_INCREMENTAL'] == '0', 'Gate concurrency/profile differs')
require(all(env.get(key) is None for key in ('RUSTFLAGS', 'RUSTC', 'RUSTUP_TOOLCHAIN', 'RUST_MIN_STACK', 'ARCADIA_CODEC_LIFECYCLE_PHASE')), 'Compiler/parent override differs')
for key, expected in {'CARGO_TARGET_DIR': ROOT + '/.local/analysis-checked-save-target-20260913', 'CARGO_HOME': ROOT + '/.local/toolchain/cargo', 'RUSTUP_HOME': ROOT + '/.local/toolchain/rustup'}.items():
require(norm(env[key]) == norm(expected), 'Gate environment path differs: ' + key)
require(norm(r['cwd']) == norm(ROOT + '/.local/as3batch') and r['invocation'] == 'cargo', 'Gate invocation/cwd differs')
require(r['inputs'] == stem + '.inputs.json' and sha(payload[root + '.inputs.json']) == r['inputs_sha256'] and sha(payload[root + '.log']) == r['log_sha256'], 'Log/input binding differs')
helper = HELPERS[source]
require(r['launcher_sha256'] == r['launcher_sha256_after'] == helper and r['launcher_copy'].replace('\\', '/') == 'launchers/' + helper + '.py' and sha(payload['gates/launchers/' + helper + '.py']) == helper, 'Launcher differs')
current = document(payload[root + '.inputs.json'])
if source not in inventories:
git_inputs(repo, source, current)
inventories[source] = current
require(current == inventories[source], 'Same-source input manifests differ')
mode = 'integration' if kind in INTEGRATIONS else ('focused' if kind == 'admission' else kind)
selected = PROJECT if kind == 'focused' else ADMISSION if kind == 'admission' else INTEGRATIONS[kind][1] if mode == 'integration' else ''
target = INTEGRATIONS[kind][0] if mode == 'integration' else None
command = {
'clippy': ['cargo', 'clippy', '--locked', '--offline', '-p', 'oa-core', '--all-targets', '--', '-D', 'warnings'],
'fmt': ['cargo', 'fmt', '--all', '--', '--check'], 'list': PREFIX + ['--', '--list'],
'full': PREFIX + ['--', '--test-threads=1'], 'focused': PREFIX + [selected, '--', '--test-threads=1'],
'integration': ['cargo', 'test', '--locked', '--offline', '-p', 'oa-core', '--test', target, selected, '--', '--exact', '--test-threads=1'],
}[mode]
require(r['mode'] == mode and r['filter'] == selected and r['test_target'] == target and r['command'] == command, 'Exact command/mode/filter differs')
require(r['exit'] == (1 if kind == 'fmt' else 101 if kind == 'full' or (source == FAILED and kind == 'focused') else 0), 'Gate exit differs')
rows = re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', log, re.M)
require(r['outcomes'] == [list(row) for row in rows], 'Receipt outcomes differ from raw log')
binary = re.findall(r'^\s*Running ([^\r\n]+) \(([^\r\n]+\.exe)\)', log, re.M)
require(len(binary) == len(r['artifacts']) == (0 if mode in ('fmt', 'clippy') else 1), 'Missing/extra actual executable')
if binary:
label, logged = binary[0]
a = r['artifacts'][0]
require(norm(logged) == norm(a['path']) and a['bytes'] > 0 and re.fullmatch('[0-9a-f]{64}', a['sha256']), 'Executable record differs')
require(label.replace('\\', '/') == ('tests/' + target + '.rs' if mode == 'integration' else 'unittests src/lib.rs'), 'Wrong executable target')
require(PureWindowsPath(a['path']).name.startswith((target or 'oa_core') + '-'), 'Wrong artifact basename')
if mode == 'list':
names = listing(log)
require(r['listing_members'] == names and r['listing_count'] == len(names) == 209 and not rows, 'Listing receipt differs')
require(set(names) == set(old_names) | set(PROJECT_NAMES) | set(ADMISSION_NAMES), 'Exact listed names differ')
else:
require(r['listing_members'] == [] and r['listing_count'] == 0, 'Non-list fabricated listing')
if mode in ('clippy', 'fmt'):
require(not rows and not re.search(r'^running \d+ tests', log, re.M), 'Unexpected tests in non-test gate')
if mode == 'clippy':
require('Finished `dev` profile' in log and not re.search(r'^error(?:\[|:)', log, re.M), 'Strict log did not close cleanly')
records[stem], logs[stem] = r, log
prior_end = None
for stem in GATES:
r = records[stem]
started, ended = datetime.fromisoformat(r['start']), datetime.fromisoformat(r['end'])
require(prior_end is None or prior_end < started, 'Serialized gate chronology differs')
prior_end = ended
for short in ('b9a31bb', '71cadfa'):
strict, listed, focused = (records[f'project-{kind}-{short}-1'] for kind in ('clippy', 'list', 'focused'))
require(datetime.fromisoformat(strict['end']) < datetime.fromisoformat(listed['start']) < datetime.fromisoformat(focused['start']), 'Strict/list/focus order differs')
require(listed['artifacts'] == focused['artifacts'], 'Focus not bound to actual listed artifact')
require(records['project-list-b9a31bb-1']['listing_members'] == records['project-list-71cadfa-1']['listing_members'], 'Successor actual listing order changed')
bad_log = logs['project-focused-b9a31bb-1']
bad = outcomes(bad_log, PROJECT_NAMES, 11, 2, 196)
require({name for name, status in bad.items() if status == 'FAILED'} == FAILED_NAMES, 'Historical Project failures changed')
for name, block in failure_blocks(bad_log, FAILED_NAMES).items():
require('assertion `left == right` failed' in block and re.search(r'left: 2\r?\n right: 1', block), 'Historical warning-count failure cause changed')
outcomes(logs['project-focused-71cadfa-1'], PROJECT_NAMES, 13, 0, 196)
outcomes(logs['project-admission-71cadfa-1'], ADMISSION_NAMES, 2, 0, 207)
current_names = records['project-list-71cadfa-1']['listing_members']
actual = outcomes(logs['project-full-71cadfa-1'], current_names, 193, 16, 0)
require(all(actual[name] == status for name, status in old_status.items()) and all(actual[name] == 'ok' for name in PROJECT_NAMES + ADMISSION_NAMES), 'Historical or added runtime status changed')
require(causes(logs['project-full-71cadfa-1'], old_causes) == old_causes, 'Exact historical failure diagnostics changed')
for kind in ('admission', 'full'):
require(records[f'project-{kind}-71cadfa-1']['artifacts'] == records['project-list-71cadfa-1']['artifacts'], 'Runtime not bound to actual listing artifact')
for kind, (target, selected) in INTEGRATIONS.items():
outcomes(logs[f'project-{kind}-71cadfa-1'], [selected], 1, 0, 4 if target == 'route_source' else 32)
for first, second in (('route-defaults', 'route-errors'), ('fragment-coordinates', 'aggregate-references')):
require(records[f'project-{first}-71cadfa-1']['artifacts'] == records[f'project-{second}-71cadfa-1']['artifacts'],
'Repeated integration target artifact differs')
test_fmt = format_blocks(logs['project-fmt-71cadfa-1'])
final_fmt = format_blocks(logs['project-fmt-aaec78f-1'])
require(len(old_fmt) == 9 and len(test_fmt) == 10 and len(final_fmt) == 8, 'Formatting block count differs')
require(all(key in old_fmt and old_fmt[key] == value and test_fmt.get(key) == value for key, value in final_fmt.items()), 'Inherited formatter block body differs')
require(set(old_fmt) - set(final_fmt) == {('crates/oa-core/src/validation.rs', 182)}, 'Baseline formatting removal differs')
require(set(test_fmt) - set(final_fmt) == {('crates/oa-core/src/language.rs', 1907), ('crates/oa-core/src/language.rs', 1917)}, 'New formatter correction scope differs')
for path in {key[0] for key in final_fmt}:
require(git(repo, 'show', BASELINE + ':' + path) == git(repo, 'show', SOURCE + ':' + path), 'Inherited formatting source blob changed')
for a, b, changed, name in ((FAILED, TESTED, 'crates/oa-core/src/language/analysis_source/mixed/project/tests.rs', 'diffs/b9a31bb-to-71cadfa.patch'), (TESTED, SOURCE, 'crates/oa-core/src/language.rs', 'diffs/71cadfa-to-aaec78f.patch')):
require(git(repo, 'diff', '--name-only', a, b).decode().splitlines() == [changed], 'Source correction scope expanded')
require(git(repo, 'diff', '--binary', a, b) == payload[name], 'Exact Git correction diff differs')
left, right = inventories[a]['inputs'], inventories[b]['inputs']
require([row['path'] for row in left] == [row['path'] for row in right] and [x['path'] for x, y in zip(left, right) if x != y] == [changed], 'Scoped input correction differs')
before = git(repo, 'show', TESTED + ':crates/oa-core/src/language.rs').decode()
after = git(repo, 'show', SOURCE + ':crates/oa-core/src/language.rs').decode()
before, after = (re.sub(r'\s', '', value) for value in (before, after))
require(before.count('ConstructedDeclarations{project,legacy_elements}') == after.count('ConstructedDeclarations{project,legacy_elements,}') == 2 and after.replace('ConstructedDeclarations{project,legacy_elements,}', 'ConstructedDeclarations{project,legacy_elements}') == before, 'Successor has non-formatting semantic delta')
return {'source': SOURCE, 'runtime_source': TESTED, 'actual_listing': 209, 'Project_pass': 13, 'admission_pass': 2,
'full_status': 'FAILED', 'full_passed': 193, 'full_failed': 16, 'exact_prior_statuses_and_diagnostics_preserved': True,
'integration_passes': 4, 'strict_status': 'PASS at runtime_source', 'successor_fmt_status': 'FAILED',
'successor_fmt_inherited_blocks': 8, 'successor_runtime_claim': False,
'limits': 'Receipt/Git consistency, not subject re-execution or authenticated authorship. Private Project only; no public source8/index/update/UI completion.'}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--repo', type=Path, required=True)
parser.add_argument('--package', type=Path, default=Path(__file__).resolve().parent)
args = parser.parse_args()
manifest = document((args.package / 'manifest.json').read_bytes())
archive_path = args.package / 'evidence.zip'
require(archive_path.stat().st_size <= 16 * 1024 * 1024, 'Compressed archive exceeds bound')
archive_raw = archive_path.read_bytes()
require(manifest['archive'] == {'file': 'evidence.zip', 'bytes': len(archive_raw), 'sha256': sha(archive_raw)},
'Top-level archive pin differs')
verifier_raw = Path(__file__).read_bytes()
require(manifest['verifier'] == {'file': 'verify.py', 'bytes': len(verifier_raw), 'sha256': sha(verifier_raw)}
and (args.package / 'verify.py').read_bytes() == verifier_raw,
'Top-level verifier pin differs')
with zipfile.ZipFile(io.BytesIO(archive_raw)) as archive:
names = archive.namelist()
require(len(names) == len(set(names)), 'Duplicate ZIP members')
require(len(names) <= 128 and sum(item.file_size for item in archive.infolist()) <= 16 * 1024 * 1024, 'Archive exceeds bounded evidence size')
for name in names:
safe_name(name)
payload = {name: archive.read(name) for name in names}
print(json.dumps(verify_payload(payload, manifest, args.repo), indent=2))
print('VERIFIED evidence consistency; full suite FAILED193/16; successor formatting FAILED8 inherited blocks; no successor runtime rerun.')
if __name__ == '__main__':
try:
main()
except (ValueError, KeyError, TypeError, OSError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
raise SystemExit('REFUSED: ' + str(error))