Freeze labelled B14 B15 budget continuations retaining completed10 and original costs
This commit is contained in:
1 parent
783608f73b
commit
7b470069f7
6 files changed
+218320
No files matched your search
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 3,
|
||||
"test_sha256": "a8ebb98bc5f1256abd956141887135c429a8991754aae055af9bf80a97cd0481",
|
||||
"copied10_all_file_hashes_pass": true,
|
||||
"old_cpu_charged_once": 259.373693711,
|
||||
"scientific_sources_unchanged": true,
|
||||
"new_geometry": 421
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 3,
|
||||
"test_sha256": "a8ebb98bc5f1256abd956141887135c429a8991754aae055af9bf80a97cd0481",
|
||||
"copied10_all_file_hashes_pass": true,
|
||||
"old_cpu_charged_once": 94.09719143599999,
|
||||
"scientific_sources_unchanged": true,
|
||||
"new_geometry": 421
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,80 @@
|
||||
"""4082 engineering continuation: immutable science, reused10, only421 new."""
|
||||
import argparse,fcntl,json,os,resource,sys,time,subprocess,signal
|
||||
from pathlib import Path
|
||||
from phase2_b1213_gate import config,coverage
|
||||
from phase2_h1_worker_v2 import checked,save,sha
|
||||
from phase2_fixed_io import BASE,freeze
|
||||
from r4bc_guard import supervise
|
||||
from s2_recovery_supervisor import tree_usage
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def validated(root):
|
||||
c=config(root)
|
||||
if c['continuation_approval']!='1794904082.797009' or c['new_indices']!=list(range(10,431)):raise PermissionError('Only approved421 never attempted')
|
||||
return c
|
||||
|
||||
def guard(root,mode,index,slot,cap):
|
||||
c=validated(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);d=root/'guards'/f'{mode}-{index:04d}';start=time.process_time()
|
||||
cmd=[sys.executable,c['scientific_entry'],mode,str(root)]+([str(index)] if mode in ['field','prep'] else [])
|
||||
result=supervise(cmd,d,dict(cpu_seconds=cap,wall_seconds=c['part_wall_seconds'] if mode in ['field','prep'] else cap+60,rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json')
|
||||
save(d/'guard-result.json',dict(mode=mode,index=index,guard=result,guard_cpu_seconds=time.process_time()-start));return 0 if result['status']=='PASS' else 1
|
||||
|
||||
def run(root):
|
||||
c=validated(root);os.sched_setaffinity(0,[c['coordinator_cpu']]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single labelled continuation')
|
||||
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
|
||||
def total():
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
|
||||
def batch(mode,ids,jobcap,geometry=False):
|
||||
nonlocal settled
|
||||
pending=list(ids)
|
||||
while pending or active:
|
||||
if requested or total()>c['total_cpu_cap']-120 or (geometry and total()>c['geometry_cpu_cap']-120) or disk_usage(root)>c['disk_bytes'] or sum(tree_usage(p.pid)[1] for p,_,_ in active.values())>32*2**30:raise RuntimeError('GLOBAL_RESOURCE_OR_SIGNAL_STOP')
|
||||
for slot,(p,index,cap) in list(active.items()):
|
||||
if p.poll() is not None:
|
||||
rr=read(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json');settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];records.append(rr);del active[slot]
|
||||
if p.returncode or rr['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
|
||||
slots=range(len(c['worker_cpus'])) if geometry else range(1)
|
||||
for slot in slots:
|
||||
if pending and slot not in active:
|
||||
cap=min(jobcap,c['total_cpu_cap']-total()-sum(z[2] for z in active.values())-300)
|
||||
if geometry:cap=min(cap,c['geometry_cpu_cap']-total()-sum(z[2] for z in active.values())-120)
|
||||
if cap<1:
|
||||
if not active:raise RuntimeError('BUDGET_RESERVATION_EXHAUSTED')
|
||||
break
|
||||
i=pending.pop(0)
|
||||
if geometry:(root/'parts'/f'{i:04d}').mkdir(parents=True,exist_ok=False)
|
||||
p=subprocess.Popen([sys.executable,'tools/phase2_gate_continuation.py','guard',str(root),mode,str(i),str(slot),str(cap)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i,cap)
|
||||
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(pid=p.pid,index=i,slot=s) for s,(p,i,_) in active.items()],new_terminal=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.2)
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_cpu_seconds=c['prior_cpu_seconds'],reused=10,new=421))
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
||||
try:
|
||||
mode='field' if c['kind']=='B14' else 'prep';batch(mode,c['new_indices'],c['part_cpu_seconds'],True)
|
||||
entries=[];results={}
|
||||
for i,row in enumerate(c['parts']):
|
||||
p=root/'parts'/f'{i:04d}'/'result.json';rr=read(p);results[row['case']]=rr;entries.append(dict(case=row['case'],index=i,result_sha256=sha(p)))
|
||||
save(root/'features-manifest.json',dict(entries=entries,all431_terminal=True,labels_read=False,reused_original10=True))
|
||||
cov={role:coverage(rows,results) for role,rows in c['rows'].items()};save(root/'coverage.json',cov)
|
||||
freeze(root,'features',[root/'contract.json',root/'features-manifest.json',root/'coverage.json',*[BASE/p for p in c['code_sha256']]])
|
||||
if c['kind']=='B14' and any(v['fraction']<.8 for v in cov.values()):
|
||||
save(root/'complete.json',dict(status='COMPLETE',decision='PARK_CHANNEL_COVERAGE',coverage=cov,cumulative_cpu_seconds=total(),records=records,models_fit=0,old_stop_preserved=True));return
|
||||
for mode in ['train','predict','score']:
|
||||
if mode=='predict':freeze(root,'model',[root/'contract.json',root/'phases/train/model.ubj',root/'phases/train/result.json',*[BASE/p for p in c['code_sha256']]])
|
||||
if mode=='score':freeze(root,'predictions',[root/'contract.json',root/'phases/predict/manifest.json',*[BASE/p for p in c['code_sha256']]])
|
||||
d=root/'phases'/mode;d.mkdir(parents=True,exist_ok=False)
|
||||
batch(mode,[0],c['phase_caps'][mode])
|
||||
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'phases/score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,models_fit=1,old_stop_preserved=True,retry=False))
|
||||
except BaseException as exc:
|
||||
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
||||
for p,_,_ in active.values():
|
||||
try:os.kill(p.pid,signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
for p,_,_ in active.values():p.wait()
|
||||
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,old_stop_preserved=True,accounting_complete=True,retry=False));raise
|
||||
finally:
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);p.add_argument('cap',nargs='?',type=float);a=p.parse_args()
|
||||
if a.action=='run':run(a.root)
|
||||
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot,a.cap))
|
||||
@@ -0,0 +1,21 @@
|
||||
import unittest,tempfile,json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import phase2_gate_continuation as m
|
||||
class ContinueTests(unittest.TestCase):
|
||||
def test_reject_replayed_completed_part(self):
|
||||
with patch.object(m,'config',return_value=dict(continuation_approval='1794904082.797009',new_indices=list(range(431)))):
|
||||
with self.assertRaises(PermissionError):m.validated(Path('/tmp'))
|
||||
def test_accept_only421_new(self):
|
||||
c=dict(continuation_approval='1794904082.797009',new_indices=list(range(10,431)))
|
||||
with patch.object(m,'config',return_value=c):self.assertIs(m.validated(Path('/tmp')),c)
|
||||
def test_guard_uses_immutable_worker_original_wall_cap(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);g=root/'guards/field-0010';g.mkdir(parents=True)
|
||||
c=dict(worker_cpus=[0],scientific_entry='tools/phase2_b14_gate.py',part_wall_seconds=120,rss_bytes=8*2**30,disk_bytes=5*2**30)
|
||||
result=dict(status='PASS',cpu_seconds=.01)
|
||||
with patch.object(m,'validated',return_value=c),patch.object(m.os,'sched_setaffinity'),patch.object(m,'supervise',return_value=result) as call:
|
||||
self.assertEqual(m.guard(root,'field',10,0,60),0)
|
||||
args=call.call_args.args;self.assertEqual(args[0][1],'tools/phase2_b14_gate.py');self.assertEqual(args[2]['wall_seconds'],120)
|
||||
self.assertEqual(json.loads((g/'guard-result.json').read_text())['guard']['status'],'PASS')
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user