77 lines
5.0 KiB
Python
77 lines
5.0 KiB
Python
"""Exactly seven never-executed graph/CDF steps, sequential, cumulative accounting."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import time
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked,read_json
|
|
from s2_recovery_supervisor import save,supervise
|
|
from s2_recovery_monitoring import disk_usage
|
|
from naive_domain_common import contract
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);limits=c['limits'];old=Path(c['original_root'])
|
|
if (root/'continuation-started.json').exists():raise FileExistsError('No repeat domain continuation')
|
|
os.sched_setaffinity(0,[16]);start=time.monotonic();done=[];reason=None
|
|
original=read_json(checked(old/'batch-complete.json',c['original_batch_sha256']))
|
|
prior=original['worker_CPU_s']+c['supplemental_audit_CPU_s']
|
|
save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(),indices=c['continuation_indices'],
|
|
contract_sha256=sha(root/'contract.json'),prior_worker_and_audit_CPU_s=prior,source_code=read_json(root/'code-freeze.json')))
|
|
for index in c['continuation_indices']:
|
|
spent=prior+sum(j['worker']['cpu_seconds'] for j in done)
|
|
if spent+limits['part_cpu_seconds']>limits['preparation_cpu_seconds']:
|
|
reason='CUMULATIVE_PREPARATION_CAP';break
|
|
if disk_usage(root)+disk_usage(old)>=limits['disk_bytes']:
|
|
reason='COMBINED_DISK_CAP';break
|
|
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
worker=supervise([sys.executable,'tools/run_naive_domain_part.py',str(root),str(index)],dest,'worker',
|
|
dict(total_cpu_seconds=limits['part_cpu_seconds']+1,job_cpu_seconds=limits['part_cpu_seconds'],
|
|
job_wall_seconds=limits['part_wall_seconds'],tree_rss_bytes=limits['part_rss_bytes'],
|
|
disk_bytes=limits['part_disk_bytes'],max_processes=1),poll=.2)
|
|
result=dest/'output/result.json'
|
|
if worker['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
|
|
worker.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
|
item=dict(index=index,case=c['cases'][index],worker=worker,result_path=str(result) if result.exists() else None,
|
|
result_sha256=sha(result) if result.exists() else None,new_decodes=0,original_failure_retained=True)
|
|
save(dest/'guard-result.json',item);done.append(item)
|
|
save(root/'progress.json',dict(completed=len(done),current_index=index,cumulative_worker_and_audit_CPU_s=prior+sum(j['worker']['cpu_seconds'] for j in done)))
|
|
print(json.dumps(dict(event='DOMAIN_CONTINUATION_TERMINAL',index=index,status=worker['status'])),flush=True)
|
|
passed=len(done)==7 and all(j['worker']['status']=='PASS' for j in done) and reason is None
|
|
save(root/'batch-complete.json',dict(status='DOMAIN_CONTINUATION_TERMINAL',passed=passed,stop_reason=reason,jobs=done,
|
|
prior_original_worker_CPU_s=original['worker_CPU_s'],supplemental_audit_CPU_s=c['supplemental_audit_CPU_s'],
|
|
continuation_worker_CPU_s=sum(j['worker']['cpu_seconds'] for j in done),
|
|
worker_CPU_s=prior+sum(j['worker']['cpu_seconds'] for j in done),
|
|
guard_and_coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
|
original_status='610_PASS_7_ORIGINAL_GUARD_FAILURES',new_decodes=0,successful_original_parts_rerun=0,
|
|
contract_sha256=sha(root/'contract.json')))
|
|
if passed:
|
|
records=[];continuations={j['index']:j for j in done}
|
|
for previous in original['jobs']:
|
|
i=previous['index'];continued=i in continuations;j=continuations[i] if continued else previous
|
|
if j['worker']['status']!='PASS':raise ValueError('Canonical join contains unsuccessful input')
|
|
checked(j['result_path'],j['result_sha256'])
|
|
records.append(dict(index=i,case=c['cases'][i],domain_continuation=continued,
|
|
original_status=previous['worker']['status'],result_path=j['result_path'],result_sha256=j['result_sha256']))
|
|
save(root/'canonical-preparation.json',dict(status='CANONICAL617_TET_DOMAIN_READY',parts=sorted(records,key=lambda r:r['index']),
|
|
original_counts=dict(PASS=610,failed=7),new_domain_steps=7,total_custody_decodes=617,new_decodes=0,
|
|
contract_sha256=sha(root/'contract.json'),original_batch_sha256=c['original_batch_sha256'],
|
|
continuation_batch_sha256=sha(root/'batch-complete.json'),domain_audit_sha256=sha(c['domain_audit']),
|
|
common_domain='all vertices of frozen retained tetrahedra, including d=0 and all positive lumped mass',
|
|
T_end='last labelled arrival time on retained tet domain; not machine end or V/P switch'))
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|