161 lines
15 KiB
Python
161 lines
15 KiB
Python
"""Collect closed #31 evidence into a fresh archive; original files stay read-only."""
|
|
from pathlib import Path
|
|
import argparse,importlib.util,json,subprocess,hashlib,sys,datetime
|
|
sys.dont_write_bytecode=True
|
|
HERE=Path(__file__).resolve().parent;ROOT=HERE.parent.parent
|
|
spec=importlib.util.spec_from_file_location('portable',ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py');h=importlib.util.module_from_spec(spec);spec.loader.exec_module(h)
|
|
PIN='17f1f89fef4a6bf520f4b09a2116548b967d030e';OLD='4fbdf6f385fa268d41074459f7a4728e5e1e9b11';BASE='bfc259854f9f7b8cdd40b8b9a043340033749b75'
|
|
REPO=ROOT/'.local/traceability-view-20260923';GATES=ROOT/'.local/traceability-view-gates-20260923'
|
|
OMIT={'.exe','.dll','.pdb','.rlib','.rmeta','.lib','.a'};SKIP={'profile','working','package','unrelated-cwd','__pycache__','artifacts'}
|
|
GAPS=set() # All three frozen gate source inventories must resolve exactly.
|
|
def validate_bundle_supplement(native):
|
|
supplement=native/'supplement-source-bundle-v2'
|
|
if (supplement/'UNACCEPTED.json').exists():raise RuntimeError('Quarantined bundle trial is not accepted evidence')
|
|
admission=h.read(supplement/'converter-admission.json')
|
|
assert admission['approved'] is True and admission['source_commit']==PIN
|
|
def verify_record(record):
|
|
raw=Path(record['path']).read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
for record in admission['records']+[admission['binary']]:verify_record(record)
|
|
admitted_paths={str(Path(r['path']).resolve()) for r in admission['records']}
|
|
builds=[folder for folder in supplement.glob('build-*') if str((folder/'summary.json').resolve()) in admitted_paths]
|
|
if not builds:raise RuntimeError('Converter admission must bind a completed build summary')
|
|
assert {folder.name for folder in builds}=={'build-01'}
|
|
for folder in builds:
|
|
summary=h.read(folder/'summary.json');assert summary['source_commit']==PIN and summary['passed'] and summary['inputs_unchanged'] and not summary['failure']
|
|
checkpoints=sorted(folder for folder in (supplement/'checkpoints').glob('*') if folder.is_dir())
|
|
if not checkpoints:raise RuntimeError('No complete source-bundle checkpoints')
|
|
assert [folder.name for folder in checkpoints]==['00-baseline','01-place','02-undo','03-redo','04-reopen']
|
|
for folder in checkpoints:
|
|
outer=h.read(folder/'bundle-receipt.json');inner=h.read(folder/'state.receipt.json')
|
|
assert outer['accepted'] is True and not outer['failure'] and outer['exit_code']==outer['converter_exit_code']==0
|
|
assert inner['source_commit']==PIN and inner['full_project_exact'] and inner['inputs_unchanged'] and not inner['failure']
|
|
assert outer['original_before']==outer['original_after']
|
|
for key,path in [('wrapper',supplement/'capture_bundle.py'),('plan',supplement/'README.md'),('converter_admission',supplement/'converter-admission.json'),('manifest',native/'input-manifest.json'),('checker',native/'check.py'),('converter_stdout',folder/'converter.stdout.log'),('converter_stderr',folder/'converter.stderr.log'),('checker_stdout',folder/'wrapper.stdout.log'),('checker_stderr',folder/'wrapper.stderr.log')]:
|
|
raw=path.read_bytes();assert outer[key]=={'bytes':len(raw),'sha256':h.digest(raw)}
|
|
for name,record in outer['snapshot_after'].items():
|
|
target=(folder/name).resolve()
|
|
if not target.is_relative_to(folder.resolve()):raise RuntimeError('Bundle path escapes checkpoint')
|
|
raw=target.read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
expected=dict(outer['original_before']);expected['converted.arcadia']=outer['converted'];expected['state.arcadia']=outer['converted']
|
|
assert outer['snapshot_after']==expected
|
|
raw=(folder/'state.receipt.json').read_bytes();assert outer['inner_receipt']=={'bytes':len(raw),'sha256':h.digest(raw)}
|
|
lifecycle_specs=[('run01-close','close','03-redo',['runs/run-01/normal-close.json','runs/run-01/launch.json']),('run02-prelaunch','prelaunch','03-redo',['runs/run-01/normal-close.json']),('run02-close','close','04-reopen',['runs/run-02/normal-close.json','runs/run-02/launch.json'])]
|
|
stamp=lambda value:datetime.datetime.fromisoformat(value.replace('Z','+00:00'))
|
|
for name,phase,checkpoint_name,event_paths in lifecycle_specs:
|
|
observation=h.read(native/'bundle-lifecycle'/f'{name}.json')
|
|
assert observation['passed'] is True and not observation['failure'] and observation['phase']==phase and observation['name']==name
|
|
expected_checkpoint=supplement/'checkpoints'/checkpoint_name/'bundle-receipt.json'
|
|
assert Path(observation['checkpoint']['path']).resolve()==expected_checkpoint.resolve()
|
|
assert {Path(r['path']).resolve() for r in observation['event_records']}=={(native/p).resolve() for p in event_paths}
|
|
assert Path(observation['recorder']['path']).resolve()==(native/'supplement-bundle-lifecycle.py').resolve()
|
|
assert Path(observation['inventory_implementation']['path']).resolve()==(supplement/'capture_bundle.py').resolve()
|
|
for record in [observation['checkpoint'],observation['recorder'],observation['inventory_implementation'],*observation['event_records']]:verify_record(record)
|
|
checkpoint=h.read(expected_checkpoint)
|
|
assert observation['actual_inventory']==observation['expected_inventory']==checkpoint['original_after']
|
|
if phase=='close':
|
|
close=h.read(native/event_paths[0]);assert stamp(observation['started_utc'])>=stamp(close['observed_exit_utc'])
|
|
else:
|
|
launch=h.read(native/'runs/run-02/launch.json');assert stamp(observation['finished_utc'])<stamp(launch['started_utc'])
|
|
return {'directory':'native/supplement-source-bundle-v2','accepted_checkpoints':[folder.name for folder in checkpoints],'rejected_preparation':'native/supplement-source-bundle','scope':'Full copied bundles and unchanged inner oracle; v1 retained as rejected preparation, not executed acceptance.'}
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--output',required=True);p.add_argument('--reviews-finalized',action='store_true');p.add_argument('--run',type=int,action='append',required=True);p.add_argument('--native',type=Path,required=True);p.add_argument('--failed-native',type=Path);p.add_argument('--review',type=Path,action='append',required=True);a=p.parse_args();NATIVE=a.native.resolve()
|
|
if not a.reviews_finalized:raise RuntimeError('Parent/reviewer final confirmation required')
|
|
out=Path(a.output).resolve()
|
|
if out.exists():raise RuntimeError('Fresh archive required')
|
|
summary=h.read(GATES/'final-03/summary.json');assert summary['source_commit']==PIN and summary['passed'] and summary['inputs_unchanged']
|
|
for attempt,pin in [('final-01',BASE),('final-02',OLD)]:
|
|
failed=h.read(GATES/attempt/'summary.json');assert failed['source_commit']==pin and not failed['passed'] and failed['inputs_unchanged']
|
|
stage=h.read(NATIVE/'staging-record.json');assert stage['source_commit']==PIN
|
|
assert h.digest((NATIVE/'input-manifest.json').read_bytes())==stage['manifest']['sha256']
|
|
assert h.read(NATIVE/'input-manifest.json')['source_commit']==PIN
|
|
failed_native=a.failed_native.resolve() if a.failed_native else None
|
|
if failed_native:
|
|
known=(ROOT/'.local/traceability-view-native-17f1f89-20260923').resolve()
|
|
if failed_native!=known or failed_native==NATIVE:raise RuntimeError('Only the explicitly reviewed initial failed stage is authorized')
|
|
failed_stage=h.read(failed_native/'staging-record.json');assert failed_stage['source_commit']==PIN
|
|
assert h.digest((failed_native/'input-manifest.json').read_bytes())==failed_stage['manifest']['sha256']
|
|
failed_launch=h.read(failed_native/'runs/run-01/launch.json');failure=h.read(failed_native/'runs/run-01/observer-failure.json')
|
|
assert failure['pid']==failed_launch['pid'] and failure['owned_process_not_killed']
|
|
assert 'traceability.arcadia' in failure['error']
|
|
if not a.run or len(set(a.run))!=len(a.run) or any(n<1 for n in a.run):raise RuntimeError('Unique explicit run IDs required')
|
|
for n in sorted(a.run):
|
|
folder=NATIVE/'runs'/f'run-{n:02d}';close=h.read(folder/'normal-close.json');launch=h.read(folder/'launch.json')
|
|
assert close['source_commit']==PIN and close['exit_code']==0 and close['immutable_inputs_unchanged'] and close['stdout']['bytes']==close['stderr']['bytes']==0
|
|
assert close['launch_sha256']==h.digest((folder/'launch.json').read_bytes()) and close['pid']==launch['pid'] and close['started_utc']==launch['started_utc']
|
|
if not {1,2}.issubset(a.run):raise RuntimeError('Both planned normally closed native runs must be included')
|
|
bundle_review=validate_bundle_supplement(NATIVE)
|
|
out.mkdir();inventory=[];source=[];available=set();seen_original=set()
|
|
def copy(src,rel):
|
|
src=Path(src).resolve();raw=src.read_bytes();dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(raw)
|
|
assert src.read_bytes()==dest.read_bytes()
|
|
item={'original':str(src),'archive':rel,'bytes':len(raw),'sha256':h.digest(raw)};inventory.append(item);available.add((len(raw),h.digest(raw)));seen_original.add(str(src));return item
|
|
def tree(folder,prefix,exclude_runs=False):
|
|
for src in sorted(folder.rglob('*')):
|
|
rel=src.relative_to(folder)
|
|
if src.is_file() and src.suffix.lower() not in OMIT and not any(s in SKIP for s in rel.parts) and not (exclude_runs and rel.parts[0]=='runs'):copy(src,prefix+'/'+rel.as_posix())
|
|
tree(GATES/'final-01','gates/failed-final-01');tree(GATES/'final-02','gates/failed-final-02');tree(GATES/'final-03','gates/final');tree(NATIVE,'native',exclude_runs=True)
|
|
if failed_native:tree(failed_native,'failed-native')
|
|
for n in sorted(a.run):tree(NATIVE/'runs'/f'run-{n:02d}','native/runs/'+f'run-{n:02d}')
|
|
for name in ('run.py','final.py','README.md'):copy(GATES/name,'gates/recipes/'+name)
|
|
if len({path.name for path in a.review})!=len(a.review):raise RuntimeError('Review filenames must be unique')
|
|
for path in a.review:copy(path,'reviews/'+path.name)
|
|
copy(ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py','verify-build.py');copy(__file__,'collect.py')
|
|
# Map each frozen gate inventory only to its own exact source commit.
|
|
# Reuse Git content rather than duplicating historical archives inside source.
|
|
blobs={}
|
|
for pin in (PIN,OLD,BASE):
|
|
paths=subprocess.check_output(['git','-c','core.longpaths=true','-C',str(REPO),'ls-tree','-rz','--name-only',pin]).decode().rstrip('\0').split('\0');blobs[pin]=dict(h.git_blobs(REPO,pin,paths))
|
|
missing=[];derived=[]
|
|
inventories=[('final-03',PIN,GATES/'final-03/inputs-before.json'),('final-02',OLD,GATES/'final-02/inputs-before.json'),('final-01',BASE,GATES/'final-01/inputs-before.json')]
|
|
for snapshot,testedpin,manifest in inventories:
|
|
for item in h.read(manifest):
|
|
path=Path(item['path']).relative_to(REPO).as_posix();match=None
|
|
for pin in [testedpin]:
|
|
if path not in blobs[pin]:continue
|
|
raw=blobs[pin][path]
|
|
for method,data in [('git',raw),('reconstructed-crlf',raw.replace(b'\r\n',b'\n').replace(b'\n',b'\r\n'))]:
|
|
if len(data)==item['bytes'] and h.digest(data)==item['sha256']:match=(pin,method,data);break
|
|
if match:break
|
|
if match is None:
|
|
raise RuntimeError('Unexpected unmapped source: '+snapshot+' '+path)
|
|
pin,method,data=match;row={'original':item['path'],'git_path':path,'commit':pin,'bytes':item['bytes'],'sha256':item['sha256'],'captured_snapshot':snapshot,'tested_base_commit':testedpin}
|
|
if method=='git':row['storage']='git'
|
|
else:
|
|
rel='source-overrides/'+snapshot+'/'+path;dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(data)
|
|
row.update(storage='override',archive=rel,derivation='Git blob LF normalized then CRLF expanded; reconstructed bytes exactly match captured receipt hash')
|
|
derived.append(row)
|
|
source.append(row);available.add((item['bytes'],item['sha256']))
|
|
# Resolve small exact referenced originals (env/runner/etc) into portable copies.
|
|
# Never copy mismatched mutable content or binary/private files to hide a gap.
|
|
for iteration in range(4):
|
|
refs=[r for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))];added=0
|
|
for r in refs:
|
|
if (r['bytes'],r['sha256']) in available or r['sha256'] in GAPS:continue
|
|
src=Path(r['path'])
|
|
if src.suffix.lower() in OMIT or any(s in {'profile','working','package','unrelated-cwd','target'} for s in src.parts):continue
|
|
if src.is_file() and str(src.resolve()) not in seen_original:
|
|
raw=src.read_bytes()
|
|
if len(raw)==r['bytes'] and h.digest(raw)==r['sha256']:
|
|
copy(src,'dependencies/'+f'{len(inventory):04d}-'+src.name);added+=1
|
|
if not added:break
|
|
h.write(out/'source-map.json',source);h.write(out/'historical-source-gaps.json',missing)
|
|
refs=[dict(receipt=item['archive'],**r) for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))]
|
|
unresolved=[r for r in refs if (r['bytes'],r['sha256']) not in available]
|
|
classified=[]
|
|
for r in unresolved:
|
|
category='omitted-binary-runtime' if Path(r['path']).suffix.lower() in OMIT else ('unavailable-failed-targeted-source' if r['sha256'] in GAPS else 'UNEXPECTED')
|
|
classified.append(dict(category=category,**r))
|
|
if any(r['category']=='UNEXPECTED' for r in classified):h.write(HERE/'unexpected-references.json',classified);raise RuntimeError('Unexpected nonbinary reference; inspect report, preserve incomplete archive')
|
|
h.write(out/'external-reference-inventory.json',classified)
|
|
h.write(out/'archive-manifest.json',{'original_receipts_unchanged':True,'files':inventory,'source_map':'source-map.json','source_map_sha256':h.digest((out/'source-map.json').read_bytes()),'source_commit':PIN,'source_bundle_supplement':bundle_review,'selected_closed_runs':sorted(a.run),'failed_native_included':bool(failed_native),'failed_native_scope':'Initial Save As include-name collision; observer failed on absent expected source. Not a passed run or inferred application crash.' if failed_native else None,'historical_source_gaps_sha256':h.digest((out/'historical-source-gaps.json').read_bytes()),'external_reference_inventory_sha256':h.digest((out/'external-reference-inventory.json').read_bytes()),'excluded':'Executables/runtime/PDB, private profile and mutable working. All three frozen source inventories must resolve exactly; no unavailable-source allowances. This archive verifies evidence, not omitted executable bytes or a runnable release.'})
|
|
(out/'.gitattributes').write_text('* -text -whitespace\n',encoding='ascii')
|
|
images=[p for p in (out/'native/runs').rglob('*') if p.suffix.lower() in ('.jpg','.jpeg','.png')]
|
|
report={'source_commit':PIN,'source_bundle_supplement':bundle_review,'selected_closed_runs':sorted(a.run),'failed_native_included':bool(failed_native),'failed_native_scope':'Initial Save As include-name collision; observer failed on absent expected source. Not a passed run or inferred application crash.' if failed_native else None,'raw_originals':len(inventory),'raw_bytes':sum(r['bytes'] for r in inventory),'source_records':len(source),'git_exact':sum(r['storage']=='git' for r in source),'reconstructed_crlf_overrides':len(derived),'historical_source_gaps':len(missing),'image_count':len(images),'distinct_images':len({h.digest(p.read_bytes()) for p in images}),'external_reference_count':len(classified),'external_unique':len({(r['bytes'],r['sha256']) for r in classified}),'external_categories':{c:sum(r['category']==c for r in classified) for c in sorted({r['category'] for r in classified})}}
|
|
h.write(HERE/(out.name+'-collection-report.json'),report);print(json.dumps(report,indent=2))
|
|
if __name__=='__main__':main()
|
|
|
|
|