50 lines
3.4 KiB
Python
50 lines
3.4 KiB
Python
from pathlib import Path
|
|
import hashlib,json,re
|
|
if not __debug__: raise RuntimeError("Assertions required")
|
|
folder=Path("C:/Users/User/Documents/Git/ARCADIA-Studio/.local/analysis-source-gates-20260913")
|
|
root=folder.parent/"analysis-core-worktree-20260911"
|
|
source="f1d79df4979121335ed7b4ce4da84b2fca8a9ef4"
|
|
def read(name): return (folder/name).read_bytes()
|
|
def data(name): return json.loads(read(name))
|
|
def sha(name): return hashlib.sha256(read(name)).hexdigest()
|
|
baseline=data("full-a2a28f8.result.json")
|
|
assert sha("full-a2a28f8.result.json")=="81e171692b5c782bd234888e94468853b47a0e54375106e4cfce831e7b58e6c1"
|
|
assert sha("full-a2a28f8.log")=="9ae08b9a0157ebef587d36b40e5f7da8dd61b13bd5594bbae659f06c75e59044"
|
|
assert baseline["exit"]==101 and baseline["changed_inputs"]==[]
|
|
result=data("full-f1d79df.result.json")
|
|
assert result["source"]==source==result["head_after"]
|
|
assert result["exit"]==101 and result["changed_inputs"]==[]
|
|
assert result["command"]==["cargo","test","--locked","-p","oa-core","--lib","--","--test-threads=1"]
|
|
listing=data("list-f1d79df.admission.json")
|
|
manifest=data("full-f1d79df.inputs.json")
|
|
assert manifest==data("list-f1d79df.inputs.json")
|
|
assert manifest["source"]==source and len(manifest["inputs"])==534
|
|
for item in manifest["inputs"]:
|
|
raw=(root/item["path"]).read_bytes()
|
|
assert len(raw)==item["bytes"] and hashlib.sha256(raw).hexdigest()==item["sha256"],item["path"]
|
|
assert result["artifacts"]==[listing["artifact"]]
|
|
artifact=Path(listing["artifact"]["path"]).read_bytes()
|
|
assert hashlib.sha256(artifact).hexdigest()==listing["artifact"]["sha256"]
|
|
def rows(name):
|
|
log=read(name).decode()
|
|
parsed=re.findall(r"^test ([^\r\n]+) \.\.\. (ok|FAILED|ignored)\r?$",log,re.M)
|
|
assert len(parsed)==len(dict(parsed))
|
|
return log,dict(parsed)
|
|
old,oldrows=rows("full-a2a28f8.log")
|
|
log,actual=rows("full-f1d79df.log")
|
|
assert len(oldrows)==142
|
|
assert len(actual)==152 and set(actual)==set(listing["names"])
|
|
assert all(actual[name]==status for name,status in oldrows.items())
|
|
new=set(actual)-set(oldrows)
|
|
assert len(new)==10 and all(name.startswith("language::analysis_source::write::tests::") and actual[name]=="ok" for name in new)
|
|
failed=sorted(name for name,status in actual.items() if status=="FAILED")
|
|
assert len(failed)==16 and failed==sorted(name for name,status in oldrows.items() if status=="FAILED")
|
|
assert re.findall(r"^test result: FAILED\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;",log,re.M)==[("136","16","0","0","0")]
|
|
blocks=re.findall(r"^---- (\S+) stdout ----\r?\n(.*?)(?=^---- |^failures:)",log,re.M|re.S)
|
|
assert sorted(name for name,body in blocks)==failed
|
|
assert all('Unsupported schema version: expected arcadia 3, 4, 5, 6 or 7.' in body for name,body in blocks)
|
|
report={"source":source,"scope":"Full suite remains FAILED; exact known-failure comparison only, not a passing full-suite admission.","cargo_exit":101,"passed":136,"failed":16,"ignored":0,"listed_outcomes":152,"verified_inputs":534,"unchanged_baseline_outcomes":142,"new_writer_passes":sorted(new),"failures":failed,"pins":{name:sha(name) for name in ["full-f1d79df.log","full-f1d79df.result.json","full-f1d79df.inputs.json","list-f1d79df.admission.json","full-a2a28f8.log","full-a2a28f8.result.json"]}}
|
|
with (folder/"full-f1d79df.comparison.json").open("x",encoding="utf8") as output: json.dump(report,output,indent=2)
|
|
print(json.dumps({key:value for key,value in report.items() if key not in ["pins","failures","new_writer_passes"]}))
|
|
|