170 lines
16 KiB
Python
170 lines
16 KiB
Python
"""Collect closed #30 evidence into a fresh archive; original files stay read-only."""
|
|
from pathlib import Path
|
|
import argparse,importlib.util,json,subprocess,hashlib,sys,datetime,copy,uuid
|
|
sys.dont_write_bytecode=True
|
|
HERE=Path(__file__).resolve().parent;ROOT=HERE.parent.parent
|
|
spec=importlib.util.spec_from_file_location('portable',ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py');h=importlib.util.module_from_spec(spec);spec.loader.exec_module(h)
|
|
PIN='301126f6098286bf440c1bd1b1ed685753984723';OLD='cf37c36be16500a4a86622540395aaafb1645f6a';BASE='17f1f89fef4a6bf520f4b09a2116548b967d030e'
|
|
REPO=ROOT/'.local/allocation-drawing-20260923';GATES=ROOT/'.local/allocation-drawing-gates-20260923'
|
|
OMIT={'.exe','.dll','.pdb','.rlib','.rmeta','.lib','.a'};SKIP={'profile','working','package','unrelated-cwd','__pycache__','artifacts','target'}
|
|
INTERRUPTED_SCOPE='Partial interrupted native trial; checkpoint evidence only. No normal-close, exit-zero, reopen acceptance, or application-crash inference.'
|
|
GAPS=set() # Both candidate inventories and reused17f converter source must resolve exactly.
|
|
def validate_native(native,coverage):
|
|
if (native/'UNACCEPTED.json').exists():raise RuntimeError('Quarantined native trial')
|
|
catalog=h.read(native/'expected/catalog.json');contract=h.read(catalog['contract']['path']);baseline=h.read(catalog['checkpoints']['B']['path'])
|
|
bindings={key:h.read(native/(key+'-binding.json')) for key in ('OA','SA')}
|
|
for key,binding in bindings.items():
|
|
rid=binding['relation_uuid'];assert str(uuid.UUID(rid))==rid and uuid.UUID(rid).version==4
|
|
first=h.read(binding['first_receipt']['path']);assert first['accepted'] and first['new_bindings'][key]==rid
|
|
assert bindings['OA']['relation_uuid']!=bindings['SA']['relation_uuid']
|
|
entries=coverage['checkpoints']
|
|
if not entries or set(x['expected'] for x in entries)!={'B','O','OS'}:raise RuntimeError('Explicit B/O/OS acceptance coverage required')
|
|
paths=[x['path'] for x in entries]
|
|
if len(paths)!=len(set(paths)):raise RuntimeError('Duplicate checkpoint')
|
|
for item in entries:
|
|
path=(native/item['path']).resolve()
|
|
if not path.is_relative_to(native):raise RuntimeError('Checkpoint outside native root')
|
|
r=h.read(path)
|
|
assert r['accepted'] is True and not r['failure'] and r['source_commit']==PIN and r['expected']==item['expected']
|
|
assert r['original_before']==r['original_after']
|
|
expected=copy.deepcopy(baseline)
|
|
for key in contract['states'][r['expected']]:
|
|
rid=bindings[key]['relation_uuid'];relation=copy.deepcopy(contract['relations'][key]);relation['id']=rid;expected['relations'][rid]=relation
|
|
assert h.read(r['actual_project']['path'])==expected
|
|
assert len(r['commands'])==2 and all(x['exit_code']==0 for x in r['commands'])
|
|
for name,record in r['snapshot_after'].items():
|
|
target=(path.parent/name).resolve()
|
|
if not target.is_relative_to(path.parent):raise RuntimeError('Snapshot path escapes checkpoint')
|
|
raw=target.read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
for record in h.records(r):
|
|
if Path(record['path']).suffix.lower() in OMIT:continue
|
|
raw=Path(record['path']).read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
for run,receipt_path in coverage['final_checkpoint_by_run'].items():
|
|
assert receipt_path in paths
|
|
checkpoint=h.read(native/receipt_path);close=h.read(native/'runs'/f'run-{int(run):02d}'/'normal-close.json')
|
|
assert checkpoint['accepted'] and checkpoint['expected']=='OS'
|
|
bundle={Path(x['path']).relative_to(native/'working').as_posix():{k:x[k] for k in ('bytes','sha256')} for x in close['working_bundle']}
|
|
assert bundle==checkpoint['original_after']
|
|
for record in close['relation_bindings_after']:
|
|
raw=Path(record['path']).read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
if set(coverage['final_checkpoint_by_run'])!={'1','2'}:raise RuntimeError('Both planned closes require explicit final OS checkpoint')
|
|
return {'accepted_checkpoints':entries,'scope':'Built-in complete bundle conversion and exact B/O/OS Project checks; only OA/SA UUID parameters, fixed once; final close coverage explicitly declared'}
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--output',required=True);p.add_argument('--reviews-finalized',action='store_true');p.add_argument('--run',type=int,action='append',required=True);p.add_argument('--native',type=Path,required=True);p.add_argument('--coverage',required=True,type=Path);p.add_argument('--converter-admission',required=True,type=Path);p.add_argument('--review',type=Path,action='append',required=True);p.add_argument('--interrupted-native',type=Path);p.add_argument('--interruption-note',type=Path);a=p.parse_args();NATIVE=a.native.resolve()
|
|
if not a.reviews_finalized:raise RuntimeError('Parent/reviewer final confirmation required')
|
|
out=Path(a.output).resolve()
|
|
if out.exists():raise RuntimeError('Fresh archive required')
|
|
summary=h.read(GATES/'final-02/summary.json');assert summary['source_commit']==PIN and summary['passed'] and summary['inputs_unchanged']
|
|
for attempt,pin in [('final-01',OLD)]:
|
|
failed=h.read(GATES/attempt/'summary.json');assert failed['source_commit']==pin and not failed['passed'] and failed['inputs_unchanged']
|
|
stage=h.read(NATIVE/'staging-record.json');assert stage['source_commit']==PIN
|
|
assert h.digest((NATIVE/'input-manifest.json').read_bytes())==stage['manifest']['sha256']
|
|
assert h.read(NATIVE/'input-manifest.json')['source_commit']==PIN
|
|
interrupted_native=a.interrupted_native.resolve() if a.interrupted_native else None
|
|
if bool(interrupted_native)!=bool(a.interruption_note):raise RuntimeError('Interrupted native root and standalone note must be supplied together')
|
|
if interrupted_native:
|
|
if interrupted_native==NATIVE:raise RuntimeError('Interrupted and accepted native roots must be distinct')
|
|
partial_stage=h.read(interrupted_native/'staging-record.json')
|
|
assert partial_stage['source_commit']==PIN and h.digest((interrupted_native/'input-manifest.json').read_bytes())==partial_stage['manifest']['sha256']
|
|
assert not list((interrupted_native/'runs').glob('*/normal-close.json'))
|
|
assert h.read(interrupted_native/'OA-binding.json')['relation_uuid']=='9dc63df6-ce83-47ee-be58-bcfa9d13482c'
|
|
for record in h.records(h.read(interrupted_native/'OA-binding.json')):
|
|
raw=Path(record['path']).read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
for checkpoint,expected in [('00-baseline','B'),('01-oa-created','O'),('02-oa-undo','B'),('03-oa-redo','O')]:
|
|
checkpoint_root=interrupted_native/'runs/run-01/checkpoints'/checkpoint;partial=h.read(checkpoint_root/'receipt.json')
|
|
assert partial['source_commit']==PIN and partial['accepted'] is True and not partial['failure'] and partial['expected']==expected and partial['original_before']==partial['original_after']
|
|
for name,record in partial['snapshot_after'].items():
|
|
target=(checkpoint_root/name).resolve();assert target.is_relative_to(checkpoint_root)
|
|
raw=target.read_bytes();assert len(raw)==record['bytes'] and h.digest(raw)==record['sha256']
|
|
if not a.run or len(set(a.run))!=len(a.run) or any(n<1 for n in a.run):raise RuntimeError('Unique explicit run IDs required')
|
|
for n in sorted(a.run):
|
|
folder=NATIVE/'runs'/f'run-{n:02d}';close=h.read(folder/'normal-close.json');launch=h.read(folder/'launch.json')
|
|
assert close['source_commit']==PIN and close['exit_code']==0 and close['immutable_inputs_unchanged'] and close['stdout']['bytes']==close['stderr']['bytes']==0
|
|
assert close['launch_sha256']==h.digest((folder/'launch.json').read_bytes()) and close['pid']==launch['pid'] and close['started_utc']==launch['started_utc']
|
|
if set(a.run)!={1,2}:raise RuntimeError('Both planned normally closed native runs must be included')
|
|
close1=h.read(NATIVE/'runs/run-01/normal-close.json');launch2=h.read(NATIVE/'runs/run-02/launch.json')
|
|
assert launch2['previous_close_sha256']==h.digest((NATIVE/'runs/run-01/normal-close.json').read_bytes())
|
|
assert launch2['working_bundle_before']==close1['working_bundle'] and launch2['relation_bindings_before']==close1['relation_bindings_after']
|
|
coverage=h.read(a.coverage);bundle_review=validate_native(NATIVE,coverage)
|
|
converter=h.read(a.converter_admission);assert converter['approved'] and converter['source_commit']==BASE
|
|
converter_inventories=[Path(r['path']) for r in converter['records'] if Path(r['path']).name in ('inputs-before.json','inputs-after.json')]
|
|
if not converter_inventories:raise RuntimeError('Raw converter source inventories required')
|
|
out.mkdir();inventory=[];source=[];available=set();seen_original=set()
|
|
def copy(src,rel):
|
|
src=Path(src).resolve();raw=src.read_bytes();dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(raw)
|
|
assert src.read_bytes()==dest.read_bytes()
|
|
item={'original':str(src),'archive':rel,'bytes':len(raw),'sha256':h.digest(raw)};inventory.append(item);available.add((len(raw),h.digest(raw)));seen_original.add(str(src));return item
|
|
def tree(folder,prefix,exclude_runs=False):
|
|
for src in sorted(folder.rglob('*')):
|
|
rel=src.relative_to(folder)
|
|
if src.is_symlink() or src.is_junction() or getattr(src.lstat(),'st_file_attributes',0)&1024:raise RuntimeError('Linked archive input')
|
|
if src.is_file() and src.suffix.lower() not in OMIT and not any(s in SKIP for s in rel.parts) and not (exclude_runs and rel.parts[0]=='runs'):copy(src,prefix+'/'+rel.as_posix())
|
|
tree(GATES/'final-01','gates/failed-final-01');tree(GATES/'final-02','gates/final');tree(NATIVE,'native',exclude_runs=True)
|
|
if interrupted_native:
|
|
tree(interrupted_native,'partial-interrupted-native')
|
|
copy(a.interruption_note,'partial-interrupted-native-note.md')
|
|
copy(a.coverage,'acceptance-coverage.json');copy(a.converter_admission,'converter-admission.json')
|
|
for n in sorted(a.run):tree(NATIVE/'runs'/f'run-{n:02d}','native/runs/'+f'run-{n:02d}')
|
|
for name in ('run.py','final.py','README.md'):copy(GATES/name,'gates/recipes/'+name)
|
|
if len({path.name for path in a.review})!=len(a.review):raise RuntimeError('Review filenames must be unique')
|
|
for path in a.review:copy(path,'reviews/'+path.name)
|
|
copy(ROOT/'.local/edge-anchors-evidence-20260923/verify-build.py','verify-build.py');copy(__file__,'collect.py')
|
|
# Map each frozen gate inventory only to its own exact source commit.
|
|
# Reuse Git content rather than duplicating historical archives inside source.
|
|
blobs={}
|
|
for pin in (PIN,OLD,BASE):
|
|
paths=subprocess.check_output(['git','-c','core.longpaths=true','-C',str(REPO),'ls-tree','-rz','--name-only',pin]).decode().rstrip('\0').split('\0');blobs[pin]=dict(h.git_blobs(REPO,pin,paths))
|
|
missing=[];derived=[]
|
|
inventories=[('final-02',PIN,GATES/'final-02/inputs-before.json',REPO),('final-01',OLD,GATES/'final-01/inputs-before.json',REPO)]+[('converter-'+str(i),BASE,path,ROOT/'.local/traceability-view-20260923') for i,path in enumerate(converter_inventories)]
|
|
for snapshot,testedpin,manifest,source_repo in inventories:
|
|
for item in h.read(manifest):
|
|
path=Path(item['path']).relative_to(source_repo).as_posix();match=None
|
|
for pin in [testedpin]:
|
|
if path not in blobs[pin]:continue
|
|
raw=blobs[pin][path]
|
|
for method,data in [('git',raw),('reconstructed-crlf',raw.replace(b'\r\n',b'\n').replace(b'\n',b'\r\n'))]:
|
|
if len(data)==item['bytes'] and h.digest(data)==item['sha256']:match=(pin,method,data);break
|
|
if match:break
|
|
if match is None:
|
|
raise RuntimeError('Unexpected unmapped source: '+snapshot+' '+path)
|
|
pin,method,data=match;row={'original':item['path'],'git_path':path,'commit':pin,'bytes':item['bytes'],'sha256':item['sha256'],'captured_snapshot':snapshot,'tested_base_commit':testedpin}
|
|
if method=='git':row['storage']='git'
|
|
else:
|
|
rel='source-overrides/'+snapshot+'/'+path;dest=out/rel;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
with dest.open('xb') as f:f.write(data)
|
|
row.update(storage='override',archive=rel,derivation='Git blob LF normalized then CRLF expanded; reconstructed bytes exactly match captured receipt hash')
|
|
derived.append(row)
|
|
source.append(row);available.add((item['bytes'],item['sha256']))
|
|
# Resolve small exact referenced originals (env/runner/etc) into portable copies.
|
|
# Never copy mismatched mutable content or binary/private files to hide a gap.
|
|
for iteration in range(4):
|
|
refs=[r for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))];added=0
|
|
for r in refs:
|
|
if (r['bytes'],r['sha256']) in available or r['sha256'] in GAPS:continue
|
|
src=Path(r['path'])
|
|
if src.suffix.lower() in OMIT or any(s in SKIP for s in src.parts):continue
|
|
if src.is_file() and str(src.resolve()) not in seen_original:
|
|
raw=src.read_bytes()
|
|
if len(raw)==r['bytes'] and h.digest(raw)==r['sha256']:
|
|
copy(src,'dependencies/'+f'{len(inventory):04d}-'+src.name);added+=1
|
|
if not added:break
|
|
h.write(out/'source-map.json',source);h.write(out/'historical-source-gaps.json',missing)
|
|
refs=[dict(receipt=item['archive'],**r) for item in inventory if item['archive'].endswith('.json') for r in h.records(h.read(out/item['archive']))]
|
|
unresolved=[r for r in refs if (r['bytes'],r['sha256']) not in available]
|
|
classified=[]
|
|
for r in unresolved:
|
|
category='omitted-binary-runtime' if Path(r['path']).suffix.lower() in OMIT else ('omitted-private-or-mutable-directory' if any(s in SKIP for s in Path(r['path']).parts) else ('unavailable-failed-targeted-source' if r['sha256'] in GAPS else 'UNEXPECTED'))
|
|
classified.append(dict(category=category,**r))
|
|
if any(r['category']=='UNEXPECTED' for r in classified):h.write(HERE/'unexpected-references.json',classified);raise RuntimeError('Unexpected nonbinary reference; inspect report, preserve incomplete archive')
|
|
h.write(out/'external-reference-inventory.json',classified)
|
|
h.write(out/'archive-manifest.json',{'original_receipts_unchanged':True,'files':inventory,'source_map':'source-map.json','source_map_sha256':h.digest((out/'source-map.json').read_bytes()),'source_commit':PIN,'source_bundle_supplement':bundle_review,'selected_closed_runs':sorted(a.run),'interrupted_native_included':bool(interrupted_native),'interrupted_native_scope':INTERRUPTED_SCOPE if interrupted_native else None,'historical_source_gaps_sha256':h.digest((out/'historical-source-gaps.json').read_bytes()),'external_reference_inventory_sha256':h.digest((out/'external-reference-inventory.json').read_bytes()),'excluded':'Executable/library/debug binaries and directories named profile, working, package, unrelated-cwd, __pycache__, artifacts, or target. Both gate and reused converter source inventories must resolve exactly; no unavailable-source allowances. This archive verifies evidence, not omitted executable bytes or a runnable release.'})
|
|
(out/'.gitattributes').write_text('* -text -whitespace\n',encoding='ascii')
|
|
images=[p for p in (out/'native/runs').rglob('*') if p.suffix.lower() in ('.jpg','.jpeg','.png')]
|
|
report={'source_commit':PIN,'source_bundle_supplement':bundle_review,'selected_closed_runs':sorted(a.run),'interrupted_native_included':bool(interrupted_native),'interrupted_native_scope':INTERRUPTED_SCOPE if interrupted_native else None,'raw_originals':len(inventory),'raw_bytes':sum(r['bytes'] for r in inventory),'source_records':len(source),'git_exact':sum(r['storage']=='git' for r in source),'reconstructed_crlf_overrides':len(derived),'historical_source_gaps':len(missing),'image_count':len(images),'distinct_images':len({h.digest(p.read_bytes()) for p in images}),'external_reference_count':len(classified),'external_unique':len({(r['bytes'],r['sha256']) for r in classified}),'external_categories':{c:sum(r['category']==c for r in classified) for c in sorted({r['category'] for r in classified})}}
|
|
h.write(HERE/(out.name+'-collection-report.json'),report);print(json.dumps(report,indent=2))
|
|
if __name__=='__main__':main()
|
|
|
|
|