211 lines
16 KiB
Python
211 lines
16 KiB
Python
"""Read-only portable S2c evidence verification; no compiler or artifact execution."""
|
|
import argparse
|
|
from datetime import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
import re
|
|
import subprocess
|
|
|
|
if not __debug__:
|
|
raise RuntimeError('Optimized Python is refused; use ordinary Python -B.')
|
|
|
|
SOURCE = 'f1d79df4979121335ed7b4ce4da84b2fca8a9ef4'
|
|
BASELINE = 'a2a28f8d23a46ba1696b6a8410d4d3b5c1808f49'
|
|
PRIOR = '355f6d4d244e6b57fd89a8afd80a2dacdea69ba1'
|
|
PRIOR_PREFIX = 'harness/experiments/analysis-source-s2b-windows-20260913/archive/evidence/.local/'
|
|
PREFIX = ['cargo', 'test', '--locked', '-p', 'oa-core', '--lib']
|
|
SCOPES = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'build', 'crates', 'assets', 'docs', 'examples', 'harness/fixtures']
|
|
WRITER = 'language::analysis_source::write::tests::'
|
|
MESSAGE = 'Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.'
|
|
|
|
|
|
def require(condition, reason):
|
|
if not condition:
|
|
raise RuntimeError(reason)
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def git(repo, *args, data=None):
|
|
return subprocess.check_output(['git', *args], cwd=repo, input=data)
|
|
|
|
|
|
def audit(repo, archive):
|
|
raw = archive / 'raw'
|
|
helpers = archive / 'helpers'
|
|
read = lambda name: (raw / name).read_bytes()
|
|
document = lambda name: json.loads(read(name))
|
|
closure = document('closure-f1d79df.json')
|
|
require(closure['source'] == SOURCE and closure['verified_inputs'] == 534, 'Closure source/input count')
|
|
require(closure['listing'] == 152 and closure['typed_pass'] == 44 and closure['raw_token_pass'] == 1 and [closure['full'][k] for k in ['passed', 'failed', 'ignored']] == [136, 16, 0], 'Closure count claims')
|
|
for record in closure['records'].values():
|
|
for name, digest in record['pins'].items():
|
|
require(sha(read(name)) == digest, 'Closure binding: ' + name)
|
|
helper_sha = sha((helpers / 'admit-analysis-s2c-gate-r2.py').read_bytes())
|
|
require(helper_sha == 'd2bd3c38feb77b8e916b3d413a278d4b0960baff8e14843aed4c2b20c26c90e1', 'Admitter differs')
|
|
require(sha((helpers / 'run-analysis-source-gate.py').read_bytes()) == 'd8d9c1c9e9dc94fdd43afb4118ebc491c8eb6d89082f8441f8395b9d4265b068', 'Recorder differs')
|
|
|
|
paths = git(repo, 'ls-tree', '-r', '--name-only', SOURCE, '--', *SCOPES).decode().splitlines()
|
|
require(len(paths) == len(set(paths)) == 534, 'Committed scope count')
|
|
batch = git(repo, 'cat-file', '--batch', data=''.join(SOURCE + ':' + p + '\n' for p in paths).encode())
|
|
offset, expected, writer_source = 0, [], None
|
|
for path in paths:
|
|
end = batch.index(b'\n', offset)
|
|
blob_id, kind, size = batch[offset:end].split()
|
|
size = int(size)
|
|
data = batch[end + 1:end + 1 + size]
|
|
require(kind == b'blob' and len(data) == size and batch[end + 1 + size:end + 2 + size] == b'\n', 'Git blob framing: ' + path)
|
|
expected.append({'path': path, 'bytes': size, 'sha256': sha(data)})
|
|
if path == 'crates/oa-core/src/language/analysis_source/write/tests.rs':
|
|
writer_source = data.decode()
|
|
offset = end + size + 2
|
|
require(offset == len(batch) and writer_source is not None, 'Complete Git batch')
|
|
new = sorted(WRITER + n for n in re.findall(r'#\[test\]\s*fn (\w+)\(', writer_source))
|
|
require(len(new) == len(set(new)) == 10, 'Ten source-derived writer cases')
|
|
|
|
previous_bytes = {}
|
|
|
|
def previous(name):
|
|
if name not in previous_bytes:
|
|
previous_bytes[name] = git(repo, 'show', PRIOR + ':' + PRIOR_PREFIX + 'analysis-source-gates-20260913/' + name)
|
|
return previous_bytes[name]
|
|
|
|
old_list = json.loads(previous('list-a2a28f8.result.json'))
|
|
old_admission = json.loads(previous('list-a2a28f8.admission.json'))
|
|
old_list_log = previous('list-a2a28f8.log').decode()
|
|
old_names = re.findall(r'^([^\r\n]+): test\r?$', old_list_log, re.M)
|
|
require(old_list['source'] == old_list['head_after'] == BASELINE and old_list['exit'] == 0 and not old_list['changed_inputs'], 'Baseline listing receipt')
|
|
require(old_names == old_list['listing_members'] == old_admission['names'] and len(old_names) == len(set(old_names)) == 142, 'Baseline actual listing')
|
|
for key, suffix in [('result_sha256', '.result.json'), ('inputs_sha256', '.inputs.json'), ('log_sha256', '.log')]:
|
|
require(old_admission[key] == sha(previous('list-a2a28f8' + suffix)), 'Baseline admission binding')
|
|
require(sha(git(repo, 'show', PRIOR + ':' + PRIOR_PREFIX + 'admit-analysis-s2b-gate.py')) == old_admission['admission_helper_sha256'], 'Baseline helper binding')
|
|
|
|
names, artifact, environment, gates, last_end = None, None, None, {}, None
|
|
logs = {}
|
|
for mode in ['list', 'typed', 'tokens', 'full', 'strict']:
|
|
stem = mode + '-f1d79df'
|
|
result = document(stem + '.result.json')
|
|
log = read(stem + '.log').decode()
|
|
logs[mode] = log
|
|
require(result['source'] == result['head_after'] == SOURCE and result['changed_inputs'] == [], 'Unchanged source: ' + mode)
|
|
require(result['inputs'] == stem + '.inputs.json' and document(result['inputs']) == {'source': SOURCE, 'inputs': expected}, 'Exact 534 Git inputs: ' + mode)
|
|
start, end = (datetime.fromisoformat(result[k]) for k in ['start', 'end'])
|
|
require(start.tzinfo is not None and end.tzinfo is not None and start <= end and (last_end is None or last_end <= start), 'Closed ordered UTC times')
|
|
last_end = end
|
|
require(type(result['exit']) is int and result['exit'] == (101 if mode == 'full' else 0), 'Actual exit: ' + mode)
|
|
recorded = closure['records'][mode]
|
|
require([recorded[k] for k in ['start', 'end', 'cargo_exit']] == [result['start'], result['end'], result['exit']], 'Closure result: ' + mode)
|
|
if environment is None:
|
|
environment = result['environment']
|
|
require(result['environment'] == environment, 'Same recorded environment')
|
|
if mode == 'strict':
|
|
require(result['command'] == ['cargo', 'clippy', '--locked', '-p', 'oa-core', '--all-targets', '--', '-D', 'warnings'], 'Strict command')
|
|
require(result['artifacts'] == [] and 'Finished `dev` profile [unoptimized]' in log and not re.search(r'^\s*(warning|error)(\[|:)', log, re.M), 'Closed clean strict log')
|
|
else:
|
|
if artifact is None:
|
|
require(len(result['artifacts']) == 1, 'One test executable')
|
|
artifact = result['artifacts'][0]
|
|
require(result['artifacts'] == [artifact], 'Same recorded test executable')
|
|
emitted = re.findall(r'Running unittests [^\r\n]+ \(([^\r\n]+\.exe)\)', log)
|
|
require(len(emitted) == 1 and PureWindowsPath(emitted[0]) == PureWindowsPath(artifact['path']), 'Raw emitted executable binding')
|
|
if mode == 'list':
|
|
require(result['command'] == PREFIX + ['--', '--list'], 'List command')
|
|
names = re.findall(r'^([^\r\n]+): test\r?$', log, re.M)
|
|
require(names == result['listing_members'] and len(names) == len(set(names)) == result['listing_count'] == 152, 'Actual unique listing')
|
|
require(set(names) == set(old_names) | set(new) and not set(old_names) & set(new), 'Exact retained142 + new10')
|
|
require(re.findall(r'^(\d+) tests, (\d+) benchmarks\s*$', log, re.M) == [('152', '0')], 'Listing summary')
|
|
else:
|
|
require(result['listing_count'] == 0 and result['listing_members'] == [], 'Non-list metadata')
|
|
if mode in ['list', 'typed', 'tokens']:
|
|
admission = document(stem + '.admission.json')
|
|
selected = names if mode == 'list' else [n for n in names if ('language::analysis_source' if mode == 'typed' else 'language::raw_tokens') in n]
|
|
require(admission['mode'] == ('list' if mode == 'list' else 'focused') and admission['selector'] == (None if mode == 'list' else 'language::analysis_source' if mode == 'typed' else 'language::raw_tokens'), 'Admission mode/selector')
|
|
require(admission['source'] == SOURCE and admission['artifact'] == artifact and admission['verified_inputs'] == 534 and admission['admission_helper_sha256'] == helper_sha, 'Admission source/helper')
|
|
require(admission['names'] == selected and admission['required_new_names'] == new and admission['baseline_source'] == BASELINE and admission['baseline_members'] == 142, 'Admission exact membership')
|
|
require(admission['baseline_result_sha256'] == sha(previous('list-a2a28f8.result.json')) and admission['baseline_admission_sha256'] == sha(previous('list-a2a28f8.admission.json')), 'Admission baseline pins')
|
|
for key, suffix in [('result_sha256', '.result.json'), ('inputs_sha256', '.inputs.json'), ('log_sha256', '.log')]:
|
|
require(admission[key] == sha(read(stem + suffix)), 'Admission raw binding')
|
|
if mode in ['typed', 'tokens', 'full']:
|
|
selector = 'language::analysis_source' if mode == 'typed' else 'language::raw_tokens'
|
|
require(result['command'] == PREFIX + ([] if mode == 'full' else [selector]) + ['--', '--test-threads=1'], 'Runtime command')
|
|
actual = outcomes(log)
|
|
selected = names if mode == 'full' else [n for n in names if selector in n]
|
|
require(list(actual) == selected, 'Exact ordered runtime membership')
|
|
passed, failed, filtered = {'typed': (44, 0, 108), 'tokens': (1, 0, 151), 'full': (136, 16, 0)}[mode]
|
|
require(list(actual.values()).count('ok') == passed and list(actual.values()).count('FAILED') == failed and 'ignored' not in actual.values(), 'Actual outcomes')
|
|
status = 'FAILED' if failed else 'ok'
|
|
require(re.findall(r'^test result: ' + status + r'\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;', log, re.M) == [(str(passed), str(failed), '0', '0', str(filtered))], 'Actual summary')
|
|
gates[mode] = {'exit': result['exit'], 'start': result['start'], 'end': result['end'], 'log_sha256': sha(read(stem + '.log')), 'result_sha256': sha(read(stem + '.result.json'))}
|
|
require(artifact == closure['test_artifact'], 'Closure executable identity')
|
|
old_log = previous('full-a2a28f8.log').decode()
|
|
old_full = json.loads(previous('full-a2a28f8.result.json'))
|
|
require(old_full['source'] == old_full['head_after'] == BASELINE and old_full['exit'] == 101 and old_full['changed_inputs'] == [], 'Baseline failed full receipt')
|
|
require(old_full['inputs'] == 'full-a2a28f8.inputs.json' and previous(old_full['inputs']) == previous('list-a2a28f8.inputs.json'), 'Baseline full/list input equality')
|
|
old_rows, rows = outcomes(old_log), outcomes(logs['full'])
|
|
require(set(old_rows) == set(old_names) and len(old_rows) == 142 and all(rows[n] == status for n, status in old_rows.items()), 'All baseline outcomes unchanged')
|
|
require(sorted(set(rows) - set(old_rows)) == new and all(rows[n] == 'ok' for n in new), 'Every new writer case passed')
|
|
old_blocks, blocks = failure_blocks(old_log), failure_blocks(logs['full'])
|
|
failed = sorted(n for n, status in rows.items() if status == 'FAILED')
|
|
require(len(failed) == 16 and sorted(blocks) == sorted(old_blocks) == failed, 'Exact sixteen panic identities')
|
|
causes = []
|
|
for name in failed:
|
|
current, old = blocks[name], old_blocks[name]
|
|
# Ignore only process/thread identities and optional one-time backtrace hint.
|
|
normalized = lambda b: [re.sub(r"' \(\d+\) panicked", "' (THREAD) panicked", line) for line in b.splitlines() if line and not line.startswith('note: run with')]
|
|
require(normalized(current) == normalized(old), 'Panic location/complete diagnostic changed: ' + name)
|
|
require(MESSAGE in current and 'line: 1, column: 9' in current and 'start: 8, end: 9' in current, 'Header cause: ' + name)
|
|
causes.append({'name': name, 'current_block_sha256': sha(current.encode()), 'baseline_block_sha256': sha(old.encode()), 'diagnostic': next(line for line in current.splitlines() if line.startswith('called `Result::unwrap()'))})
|
|
comparison = document('full-f1d79df.comparison.json')
|
|
require(comparison['source'] == SOURCE and comparison['cargo_exit'] == 101 and comparison['passed'] == 136 and comparison['failed'] == 16 and comparison['ignored'] == 0 and comparison['failures'] == failed and comparison['new_writer_passes'] == new and comparison['unchanged_baseline_outcomes'] == 142 and comparison['verified_inputs'] == 534 and comparison['listed_outcomes'] == 152, 'Comparison receipt facts')
|
|
for name, digest in comparison['pins'].items():
|
|
require(sha(previous(name) if 'a2a28f8' in name else read(name)) == digest, 'Comparison binding: ' + name)
|
|
failure_sources = ['crates/oa-core/src/' + p for p in ['model.rs', 'language.rs', 'language_nesting_tests.rs', 'source_project.rs', 'source_project/source_migration.rs', 'source_files.rs']]
|
|
require(git(repo, 'ls-tree', '-r', SOURCE, '--', *failure_sources) == git(repo, 'ls-tree', '-r', BASELINE, '--', *failure_sources), 'Failure-path source blobs changed')
|
|
return {'source': SOURCE, 'baseline_source': BASELINE, 'prior_evidence_commit': PRIOR, 'verified_git_inputs': 534, 'listing': 152, 'typed_pass': 44, 'raw_token_pass': 1, 'full': {'passed': 136, 'failed': 16, 'ignored': 0, 'exit': 101}, 'strict_exit': 0, 'new_writer_passes': new, 'gates': gates, 'recorded_test_artifact': artifact, 'failure_causes': causes, 'unchanged_failure_sources': failure_sources, 'scope': 'Private staged writer evidence; full suite FAILED. No public8, S3/S4, native or release acceptance.'}
|
|
|
|
|
|
def outcomes(log):
|
|
rows = re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$', log, re.M)
|
|
require(len(rows) == len(dict(rows)), 'Duplicate raw outcome')
|
|
return dict(rows)
|
|
|
|
|
|
def failure_blocks(log):
|
|
rows = re.findall(r'^---- (\S+) stdout ----\r?\n(.*?)(?=^---- |^failures:)', log, re.M | re.S)
|
|
require(len(rows) == len(dict(rows)), 'Duplicate panic section')
|
|
return dict(rows)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--repo', required=True, type=Path, help='Git repository containing both immutable source/evidence commits')
|
|
args = parser.parse_args()
|
|
archive = Path(__file__).resolve().parent
|
|
manifest = json.loads((archive / 'manifest.json').read_bytes())
|
|
require(manifest['source'] == SOURCE and manifest['prior_evidence_commit'] == PRIOR, 'Manifest source pins')
|
|
rows = manifest['files']
|
|
paths = [r['path'] for r in rows]
|
|
require(len(paths) == len(set(paths)), 'Duplicate manifest member')
|
|
actual = {p.relative_to(archive).as_posix() for p in archive.rglob('*') if p.is_file()}
|
|
require(actual == set(paths) | {'manifest.json'}, 'Exact archive inventory')
|
|
for row in rows:
|
|
path = PurePosixPath(row['path'])
|
|
require(not path.is_absolute() and '..' not in path.parts and '\\' not in row['path'], 'Relative member required')
|
|
file = archive / row['path']
|
|
require(not file.is_symlink(), 'No linked payload')
|
|
data = file.read_bytes()
|
|
require(len(data) == row['bytes'] and sha(data) == row['sha256'], 'Payload hash: ' + row['path'])
|
|
for row in manifest['external_refs']:
|
|
data = git(args.repo, 'show', row['commit'] + ':' + row['path'])
|
|
require(len(data) == row['bytes'] and sha(data) == row['sha256'], 'Immutable prior evidence binding')
|
|
result = audit(args.repo, archive)
|
|
require(result == json.loads((archive / 'audit.json').read_bytes()), 'Recomputed independent audit differs')
|
|
print(json.dumps({'verified_payloads': len(rows), 'verified_git_inputs': 534, 'listing': 152, 'typed_pass': 44, 'token_pass': 1, 'full': result['full'], 'strict_exit': 0, 'source': SOURCE}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|