117 lines
7.5 KiB
Python
117 lines
7.5 KiB
Python
"""Read-only independent evidence audit; does not run project code or admissions."""
|
|
from pathlib import Path
|
|
import ast, hashlib, json, re, subprocess
|
|
|
|
base = Path(__file__).resolve().parent.parent
|
|
repo = base / '.local/as2-20260913'
|
|
folder = base / '.local/analysis-source-gates-20260913'
|
|
pin = 'b2c970b4673e062e47eadd12906d2316c4b1a7b3'
|
|
def sha(raw): return hashlib.sha256(raw).hexdigest()
|
|
def read(name, suffix): return (folder / (name + suffix)).read_bytes()
|
|
def obj(name, suffix='.result.json'): return json.loads(read(name, suffix))
|
|
def outcomes(log): return re.findall(r'^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\s*$', log, re.M)
|
|
|
|
# Independently rebuild the runner's declared tracked-input universe from Git.
|
|
scopes = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'build', 'crates', 'assets', 'docs', 'examples', 'harness/fixtures']
|
|
paths = subprocess.check_output(['git', 'ls-tree', '-r', '--name-only', pin, *scopes], cwd=repo).decode().splitlines()
|
|
assert len(paths) == len(set(paths)) == 524
|
|
raw = subprocess.check_output(['git', 'cat-file', '--batch'], cwd=repo, input=''.join(f'{pin}:{p}\n' for p in paths).encode())
|
|
offset, expected = 0, []
|
|
for path in paths:
|
|
end = raw.index(b'\n', offset)
|
|
_, kind, size = raw[offset:end].split()
|
|
assert kind == b'blob'
|
|
size = int(size)
|
|
blob = raw[end+1:end+1+size]
|
|
expected.append({'path': path, 'bytes': size, 'sha256': sha(blob)})
|
|
offset = end + size + 2
|
|
assert offset == len(raw)
|
|
listing = obj('list-b2c970b')
|
|
listed = re.findall(r'^([^\r\n]+): test\r?$', read('list-b2c970b', '.log').decode(), re.M)
|
|
assert listed == listing['listing_members']
|
|
assert len(listed) == len(set(listed)) == listing['listing_count'] == 129
|
|
original = obj('list-6216cff')['listing_members']
|
|
assert len(original) == len(set(original)) == 118 and set(original) <= set(listed)
|
|
new = set(listed) - set(original)
|
|
assert len(new) == 11
|
|
helper = (base / '.local/admit-analysis-s2a-gate.py').read_bytes()
|
|
tree = ast.parse(helper)
|
|
declared_new = next(ast.literal_eval(n.value) for n in ast.walk(tree) if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'new_names' for t in n.targets))
|
|
assert new == declared_new
|
|
|
|
results, artifact_identity = {}, None
|
|
for name in ['list-b2c970b', 'typed-b2c970b', 'tokens-b2c970b', 'full-b2c970b']:
|
|
rec = obj(name)
|
|
assert rec['source'] == rec['head_after'] == pin and not rec['changed_inputs']
|
|
assert rec['inputs'] == name + '.inputs.json'
|
|
manifest = obj(name, '.inputs.json')
|
|
assert manifest['source'] == pin and manifest['inputs'] == expected
|
|
assert len(rec['artifacts']) == 1
|
|
artifact = rec['artifacts'][0]
|
|
content = Path(artifact['path']).read_bytes()
|
|
assert len(content) == artifact['bytes'] and sha(content) == artifact['sha256']
|
|
if artifact_identity is None: artifact_identity = artifact
|
|
assert artifact_identity == artifact
|
|
log = read(name, '.log').decode()
|
|
raw_artifacts = re.findall(r'Running unittests [^\r\n]+ \(([^\r\n]+\.exe)\)', log)
|
|
assert len(raw_artifacts) == 1
|
|
actual_path = Path(raw_artifacts[0])
|
|
if not actual_path.is_absolute(): actual_path = base / '.local/analysis-core-worktree-20260911' / actual_path
|
|
assert actual_path.resolve() == Path(artifact['path']).resolve()
|
|
rows = outcomes(log)
|
|
assert len(rows) == len(set(n for n, _ in rows))
|
|
results[name] = {'exit': rec['exit'], 'outcomes': dict(rows), 'result_sha256': sha(read(name,'.result.json')), 'inputs_sha256': sha(read(name,'.inputs.json')), 'log_sha256': sha(read(name,'.log'))}
|
|
if name != 'full-b2c970b':
|
|
assert rec['exit'] == 0
|
|
receipt = obj(name, '.admission.json')
|
|
assert receipt['source'] == pin and receipt['artifact'] == artifact and receipt['verified_inputs'] == 524
|
|
assert receipt['admission_helper_sha256'] == sha(helper)
|
|
for k in ['result_sha256', 'inputs_sha256', 'log_sha256']: assert receipt[k] == results[name][k]
|
|
if name == 'list-b2c970b':
|
|
assert receipt['mode'] == 'list' and receipt['names'] == listed
|
|
assert re.findall(r'^(\d+) tests, (\d+) benchmarks\s*$', log, re.M) == [('129','0')]
|
|
else:
|
|
selector = 'language::analysis_source' if name.startswith('typed') else 'language::raw_tokens'
|
|
members = [n for n in listed if selector in n]
|
|
assert selector in rec['command'] and receipt['mode'] == 'focused' and receipt['names'] == members
|
|
assert set(dict(rows)) == set(members) and all(s == 'ok' for n,s in rows)
|
|
assert re.search(rf'test result: ok\. {len(rows)} passed; 0 failed; 0 ignored;',log)
|
|
|
|
full = results['full-b2c970b']['outcomes']
|
|
assert set(full) == set(listed)
|
|
assert results['full-b2c970b']['exit'] == 101
|
|
assert list(full.values()).count('ok') == 113 and list(full.values()).count('FAILED') == 16
|
|
assert re.search(r'test result: FAILED\. 113 passed; 16 failed; 0 ignored; 0 measured; 0 filtered out;', read('full-b2c970b','.log').decode())
|
|
failures = {n for n,s in full.items() if s == 'FAILED'}
|
|
old_log = read('full-2c515cf','.log').decode()
|
|
old_rows = outcomes(old_log)
|
|
old_failed = {n for n,s in old_rows if s == 'FAILED'}
|
|
assert failures == old_failed and len(old_failed) == 16
|
|
assert len(old_rows) == len(set(n for n,s in old_rows)) == 114
|
|
assert list(dict(old_rows).values()).count('ok') == 98
|
|
retained = [n for n in listed if n.startswith('language::source3_bridge::tests::') or n.startswith('persistence::source_save_tests::')]
|
|
assert len(retained) == 8 and all(full[n] == 'ok' for n in retained)
|
|
assert all(full[n] == 'ok' for n in new)
|
|
def failure_sections(log):
|
|
return {n: body for n,body in re.findall(r'^---- (\S+) stdout ----\r?\n(.*?)(?=^---- |^failures:|\Z)',log,re.M|re.S)}
|
|
sections = failure_sections(read('full-b2c970b','.log').decode())
|
|
prior_sections = failure_sections(old_log)
|
|
assert set(sections) == failures
|
|
assert all('Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.' in body for body in sections.values())
|
|
changed_cause = [n for n in failures if 'Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.' not in prior_sections[n]]
|
|
assert len(changed_cause) == 2 and all('definitions' in prior_sections[n] for n in changed_cause)
|
|
|
|
# Inspect strict exactly once if its result already exists; never poll a running gate.
|
|
strict = {'state': 'result absent at audit; not inspected'}
|
|
strict_path = folder/'strict-b2c970b.result.json'
|
|
if strict_path.exists():
|
|
rec = obj('strict-b2c970b')
|
|
manifest = obj('strict-b2c970b','.inputs.json')
|
|
assert rec['source'] == rec['head_after'] == pin and not rec['changed_inputs']
|
|
assert manifest['source'] == pin and manifest['inputs'] == expected
|
|
strict = {'exit':rec['exit'], 'command':rec['command'], 'result_sha256':sha(strict_path.read_bytes()), 'log_sha256':sha(read('strict-b2c970b','.log')), 'log_tail':read('strict-b2c970b','.log').decode()[-1500:]}
|
|
report = {'source':pin,'git_blob_verified_inputs':524,'listing_unique':129,'full':{'exit':101,'passed':113,'failed':16,'failures':sorted(failures),'same_failure_names_as_2c515cf':True,'changed_cause_from_legacy_definitions_to_header':sorted(changed_cause)},'new11_all_passed':sorted(new),'retained8_all_passed':retained,'typed_passed':len(results['typed-b2c970b']['outcomes']),'tokens_passed':len(results['tokens-b2c970b']['outcomes']),'artifact':artifact_identity,'records':{n:{k:v for k,v in r.items() if k!='outcomes'} for n,r in results.items()},'strict':strict,'admission_helper_sha256':sha(helper)}
|
|
dest = base/'.local/analysis-source8-s2a-execution-audit-20260913.json'
|
|
with dest.open('x',encoding='utf-8') as stream: json.dump(report,stream,indent=2)
|
|
print(json.dumps(report,indent=2))
|