50 lines
3.2 KiB
Python
50 lines
3.2 KiB
Python
"""One sequential, non-resumable ten-part attempt with shared 3h/5GiB limits."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
from s2_e1_supervisor import supervise,save,disk_usage
|
|
from freeze_g1_components import sha
|
|
|
|
|
|
def remaining_limits(elapsed, cpu, guard_cpu):
|
|
wall=10800-elapsed;total_worker_cpu=10800-guard_cpu
|
|
if wall<=0 or total_worker_cpu-cpu<=0:return None
|
|
return dict(total_cpu_seconds=total_worker_cpu,job_cpu_seconds=min(1800,total_worker_cpu-cpu),
|
|
job_wall_seconds=min(1800,wall),tree_rss_bytes=1342177280,disk_bytes=5*2**30,max_processes=1)
|
|
|
|
|
|
def run(root):
|
|
contract=json.loads((root/'contract.json').read_text())
|
|
for p,h in contract['code_sha256'].items():assert sha(p)==h,p
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
with (root/'batch-started.json').open('x') as f:json.dump(dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json')),f)
|
|
start=time.monotonic();cpu_start=time.process_time();worker_cpu=0.;rows=[];stop_reason=None
|
|
for i,spec in enumerate(contract['cases']):
|
|
limits=remaining_limits(time.monotonic()-start,worker_cpu,time.process_time()-cpu_start)
|
|
if limits is None:stop_reason='total_wall_or_cpu_budget_exhausted';break
|
|
if disk_usage(root)>=5*2**30:stop_reason='total_disk_budget_exhausted';break
|
|
name=f'part-{i:02d}'
|
|
r=supervise([sys.executable,'tools/s2_preprocess.py',str(root),str(i)],root,name,limits,poll=.1)
|
|
worker_cpu+=r['cpu_seconds'];row=dict(index=i,case=spec['case'],**r);rows.append(row)
|
|
print(json.dumps({k:row[k] for k in ['index','case','status','stop_reason','cpu_seconds','wall_seconds','peak_tree_rss_bytes']}),flush=True)
|
|
save(root/'batch-progress.json',dict(completed_attempts=len(rows),worker_cpu_seconds=worker_cpu,supervisor_cpu_seconds=time.process_time()-cpu_start,
|
|
wall_seconds=time.monotonic()-start,last=row))
|
|
unrun=[dict(index=i,case=contract['cases'][i]['case'],status='NOT_RUN_BUDGET') for i in range(len(rows),10)]
|
|
data=dict(status='ATTEMPTS_COMPLETE' if len(rows)==10 else 'STOPPED_AT_STAGE_BUDGET',attempts=rows,not_run=unrun,
|
|
stop_reason=stop_reason,failed_attempts=sum(r['status']!='PASS' for r in rows),retry=False,replacement=False,
|
|
worker_cpu_seconds=worker_cpu,supervisor_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
|
|
peak_worker_tree_rss_bytes=max((r['peak_tree_rss_bytes'] for r in rows),default=0),disk_bytes=disk_usage(root),
|
|
contract_sha256=sha(root/'contract.json'),labels_read=False,training_gpu=False)
|
|
if data['wall_seconds']>10800:data['stage_wall_budget_exceeded']=True
|
|
if worker_cpu+data['supervisor_cpu_seconds']>10800:data['stage_cpu_budget_exceeded']=True
|
|
save(root/'batch-complete.json',data)
|
|
print(json.dumps({k:v for k,v in data.items() if k not in ['attempts','not_run']}),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|