95 lines
8.7 KiB
Python
95 lines
8.7 KiB
Python
"""Collect closed #23 evidence into a fresh archive; original files stay read-only."""
|
|
from pathlib import Path
|
|
import argparse,importlib.util,json,subprocess,hashlib
|
|
HERE=Path(__file__).resolve().parent;ROOT=HERE.parent.parent
|
|
spec=importlib.util.spec_from_file_location('portable',ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py');h=importlib.util.module_from_spec(spec);spec.loader.exec_module(h)
|
|
PIN='b8685416f56e53e30bd11ecf93a54cea10ac8458';OLD='26979c109384add3e2a4bfd0092b8da1464e37a2';BASE='53d01a3d4451b94792986f2dfac56f6f04e111a4'
|
|
REPO=ROOT/'.local/state-bends-20260923';GATES=ROOT/'.local/state-bends-gates-20260923';NATIVE=ROOT/'.local/state-bends-native-b868541-20260923'
|
|
OMIT={'.exe','.dll','.pdb'};SKIP={'profile','working','package','unrelated-cwd','__pycache__','artifacts'}
|
|
GAPS={'97dd304bf6a1fffe14d4f0780ea66f6df9f14120fbb16a4869f02c75063f7e58','77fe84cfd420de45e8e2bc389b776e49662fb83e893f8b9d5d9fced5b8468d47'}
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--output',required=True);p.add_argument('--reviews-finalized',action='store_true');p.add_argument('--run',type=int,action='append',required=True);a=p.parse_args()
|
|
if not a.reviews_finalized:raise RuntimeError('Parent/reviewer final confirmation required')
|
|
out=Path(a.output).resolve()
|
|
if out.exists():raise RuntimeError('Fresh archive required')
|
|
summary=h.read(GATES/'final-02/summary.json');assert summary['source_commit']==PIN and summary['passed'] and summary['inputs_unchanged']
|
|
if not a.run or len(set(a.run))!=len(a.run) or any(n<1 for n in a.run):raise RuntimeError('Unique explicit run IDs required')
|
|
for n in sorted(a.run):
|
|
folder=NATIVE/'runs'/f'run-{n:02d}';close=h.read(folder/'normal-close.json');launch=h.read(folder/'launch.json')
|
|
assert close['source_commit']==PIN and close['exit_code']==0 and close['immutable_inputs_unchanged'] and close['stdout']['bytes']==close['stderr']['bytes']==0
|
|
assert close['launch_sha256']==h.digest((folder/'launch.json').read_bytes()) and close['pid']==launch['pid'] and close['started_utc']==launch['started_utc']
|
|
out.mkdir();inventory=[];source=[];available=set();seen_original=set()
|
|
def copy(src,rel):
|
|
src=Path(src).resolve();raw=src.read_bytes();dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(raw)
|
|
assert src.read_bytes()==dest.read_bytes()
|
|
item={'original':str(src),'archive':rel,'bytes':len(raw),'sha256':h.digest(raw)};inventory.append(item);available.add((len(raw),h.digest(raw)));seen_original.add(str(src));return item
|
|
def tree(folder,prefix,exclude_runs=False):
|
|
for src in sorted(folder.rglob('*')):
|
|
rel=src.relative_to(folder)
|
|
if src.resolve()==(NATIVE/'supplement-moved-reopen/moved.arcadia').resolve():continue
|
|
if src.is_file() and src.suffix.lower() not in OMIT and not any(s in SKIP for s in rel.parts) and not (exclude_runs and rel.parts[0]=='runs'):copy(src,prefix+'/'+rel.as_posix())
|
|
tree(GATES/'targeted-02','gates/failed-targeted-02');tree(GATES/'final-01','gates/failed-final-01');tree(GATES/'final-02','gates/final');tree(NATIVE,'native',exclude_runs=True)
|
|
for n in sorted(a.run):tree(NATIVE/'runs'/f'run-{n:02d}','native/runs/'+f'run-{n:02d}')
|
|
for name in ('run.py','final.py','final-README.md'):copy(GATES/name,'gates/recipes/'+name)
|
|
review_names=['acceptance-review','source-review','pan-fix-review','final02-gate-review','stage-review','native-review','native-report','integration-review']
|
|
for name in review_names:copy(ROOT/'.local'/('state-bends-'+name+'-20260923.md'),'reviews/'+name+'.md')
|
|
copy(ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py','verify-build.py');copy(__file__,'collect.py')
|
|
# Resolve actual historical source bytes. Dirty failed snapshot may match later
|
|
# committed bytes, but that is storage provenance, never its tested source pin.
|
|
blobs={}
|
|
for pin in (PIN,OLD,BASE):
|
|
paths=subprocess.check_output(['git','-c','core.longpaths=true','-C',str(REPO),'ls-tree','-rz','--name-only',pin]).decode().rstrip('\0').split('\0');blobs[pin]=dict(h.git_blobs(REPO,pin,paths))
|
|
missing=[];derived=[]
|
|
inventories=[('final-02',PIN,GATES/'final-02/inputs-before.json'),('final-01',OLD,GATES/'final-01/inputs-before.json'),('targeted-02',BASE,GATES/'targeted-02/inputs-before.json')]
|
|
for snapshot,testedpin,manifest in inventories:
|
|
for item in h.read(manifest):
|
|
path=Path(item['path']).relative_to(REPO).as_posix();match=None
|
|
for pin in ([testedpin] if snapshot!='targeted-02' else [BASE,OLD,PIN]):
|
|
if path not in blobs[pin]:continue
|
|
raw=blobs[pin][path]
|
|
for method,data in [('git',raw),('reconstructed-crlf',raw.replace(b'\r\n',b'\n').replace(b'\n',b'\r\n'))]:
|
|
if len(data)==item['bytes'] and h.digest(data)==item['sha256']:match=(pin,method,data);break
|
|
if match:break
|
|
if match is None:
|
|
if snapshot=='targeted-02' and item['sha256'] in GAPS:missing.append(dict(snapshot=snapshot,reason='Failed diagnostic untracked source not preserved',**item));continue
|
|
raise RuntimeError('Unexpected unmapped source: '+snapshot+' '+path)
|
|
pin,method,data=match;row={'original':item['path'],'git_path':path,'commit':pin,'bytes':item['bytes'],'sha256':item['sha256'],'captured_snapshot':snapshot,'tested_base_commit':testedpin}
|
|
if method=='git':row['storage']='git'
|
|
else:
|
|
rel='source-overrides/'+snapshot+'/'+path;dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(data)
|
|
row.update(storage='override',archive=rel,derivation='Git blob LF normalized then CRLF expanded; reconstructed bytes exactly match captured receipt hash')
|
|
derived.append(row)
|
|
source.append(row);available.add((item['bytes'],item['sha256']))
|
|
# Resolve small exact referenced originals (env/runner/etc) into portable copies.
|
|
# Never copy mismatched mutable content or binary/private files to hide a gap.
|
|
for iteration in range(4):
|
|
refs=[r for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))];added=0
|
|
for r in refs:
|
|
if (r['bytes'],r['sha256']) in available or r['sha256'] in GAPS:continue
|
|
src=Path(r['path'])
|
|
if src.suffix.lower() in OMIT or any(s in {'profile','working','package','unrelated-cwd','target'} for s in src.parts):continue
|
|
if src.is_file() and str(src.resolve()) not in seen_original:
|
|
raw=src.read_bytes()
|
|
if len(raw)==r['bytes'] and h.digest(raw)==r['sha256']:
|
|
copy(src,'dependencies/'+f'{len(inventory):04d}-'+src.name);added+=1
|
|
if not added:break
|
|
h.write(out/'source-map.json',source);h.write(out/'historical-source-gaps.json',missing)
|
|
refs=[dict(receipt=item['archive'],**r) for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))]
|
|
unresolved=[r for r in refs if (r['bytes'],r['sha256']) not in available]
|
|
classified=[]
|
|
for r in unresolved:
|
|
category='omitted-binary-runtime' if Path(r['path']).suffix.lower() in OMIT else ('unavailable-failed-targeted-source' if r['sha256'] in GAPS else 'UNEXPECTED')
|
|
classified.append(dict(category=category,**r))
|
|
if any(r['category']=='UNEXPECTED' for r in classified):h.write(HERE/'unexpected-references.json',classified);raise RuntimeError('Unexpected nonbinary reference; inspect report, preserve incomplete archive')
|
|
h.write(out/'external-reference-inventory.json',classified)
|
|
h.write(out/'archive-manifest.json',{'original_receipts_unchanged':True,'files':inventory,'source_map':'source-map.json','source_map_sha256':h.digest((out/'source-map.json').read_bytes()),'source_commit':PIN,'selected_closed_runs':sorted(a.run),'historical_source_gaps_sha256':h.digest((out/'historical-source-gaps.json').read_bytes()),'external_reference_inventory_sha256':h.digest((out/'external-reference-inventory.json').read_bytes()),'excluded':'Executables/runtime/PDB, private profile and mutable working. Two unavailable failed-targeted untracked source versions explicitly recorded; final successful source complete.'})
|
|
(out/'.gitattributes').write_text('* -text -whitespace\n',encoding='ascii')
|
|
images=[p for p in (out/'native/runs').rglob('*') if p.suffix.lower() in ('.jpg','.jpeg','.png')]
|
|
report={'source_commit':PIN,'selected_closed_runs':sorted(a.run),'raw_originals':len(inventory),'raw_bytes':sum(r['bytes'] for r in inventory),'source_records':len(source),'git_exact':sum(r['storage']=='git' for r in source),'reconstructed_crlf_overrides':len(derived),'historical_source_gaps':len(missing),'image_count':len(images),'distinct_images':len({h.digest(p.read_bytes()) for p in images}),'external_reference_count':len(classified),'external_unique':len({(r['bytes'],r['sha256']) for r in classified}),'external_categories':{c:sum(r['category']==c for r in classified) for c in sorted({r['category'] for r in classified})}}
|
|
h.write(HERE/(out.name+'-collection-report.json'),report);print(json.dumps(report,indent=2))
|
|
if __name__=='__main__':main()
|
|
|
|
|