Prepare amended phase3 item1 six-fit entry with complete schedules and120000GPU-s cap
This commit is contained in:
1 parent
30c34b0e66
commit
ae089067a1
23 files changed
+3129
-1
No files matched your search
@@ -0,0 +1,70 @@
|
||||
"""Item1 frozen roles, schedules and costs. No model execution or target access."""
|
||||
import json,math
|
||||
from pathlib import Path
|
||||
from phase2_fixed_io import BASE
|
||||
from phase2_h1_worker_v2 import checked,sha
|
||||
from phase2_home_freeze_io import verify
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def lr_step(step,total=64000,warmup=3200):
|
||||
if not 1<=step<=total or not 1<warmup<total:raise ValueError('1-based fixed update domain')
|
||||
if step<=warmup:return 1e-4+(step-1)*9e-4/(warmup-1)
|
||||
return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(step-warmup)/(total-warmup)))/2
|
||||
|
||||
def execution_command(args):
|
||||
return any(Path(x).name in {'phase3_data_sufficiency_v2_queue.py','phase3_data_sufficiency_v2_guard.py','phase3_data_sufficiency_v2_worker.py'} for x in args)
|
||||
|
||||
def config(root):
|
||||
c=read(root/'contract.json');verify(root,'implementation')
|
||||
if c['approval']!='1794948643.973609' or c['new_fits']!=6 or c['item_gpu_cap']!=120000 or c['budget_approval']!='1794948710.366709' or c['phase3_gpu_cap']!=612000 or c['train_gpu_cap']+c['postfit_gpu_reserve']!=c['item_gpu_cap']:raise PermissionError('Frozen item1 scope')
|
||||
if c['U']!=32000 or c['delta']!=.001 or c['seed']!=0 or c['fold']!=0 or not c['no_retry']:raise ValueError('Frozen science identity')
|
||||
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
||||
pilot=read(c['pilot_contract'])
|
||||
import torch,numpy as np
|
||||
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Original library identity')
|
||||
if len({j['gpu_uuid'] for j in c['jobs']})!=4 or sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Job resources')
|
||||
pools=c['pools'];sets=[set(pools[str(p)]['learning']) for p in [25,50,100]]
|
||||
if not sets[0]<sets[1]<sets[2] or sets[2]!=set(pilot['folds'][0]['learning']):raise ValueError('Original nested roles')
|
||||
for p in [25,50,100]:
|
||||
q=pools[str(p)]
|
||||
if set(q['learning'])&(set(q['validation'])|set(q['test'])) or q['validation']!=pilot['folds'][0]['validation'] or q['test']!=pilot['folds'][0]['test']:raise ValueError('Role separation')
|
||||
if read(q['clock_path'])['parts']!=q['learning'] or q['learning_nodes']!=sum(pilot['parts'][x]['nodes'] for x in q['learning']):raise ValueError('Learning clock/node identity')
|
||||
for j in c['jobs']:
|
||||
q=pools[str(j['pool'])];target=200*len(q['learning']) if j['mode']=='epoch' else j['step_budget']
|
||||
if j['target_steps']!=target or j['learning_parts']!=len(q['learning']) or j['learning_nodes']!=q['learning_nodes']:raise ValueError('Cell identity')
|
||||
return c,pilot
|
||||
|
||||
def view_for(c,pilot,job):return dict(pilot,folds=[c['pools'][str(job['pool'])]])
|
||||
|
||||
def project_cell(job,observation):
|
||||
v=observation;steps=v['steps'];per=v['max_epoch_seconds']/job['learning_parts'];val=max(v['max_validation_seconds'],job['validation_seconds_bound'])
|
||||
if not 0<steps<=job['target_steps'] or not all(math.isfinite(x) and x>=0 for x in [per,val,v['charged']]) or per<=0:raise ValueError('Invalid cost evidence')
|
||||
checks=20 if job['mode']=='epoch' else job['target_steps']//3200
|
||||
future=1.25*((job['target_steps']-steps)*per+checks*val)+120
|
||||
return dict(id=job['id'],pool=job['pool'],mode=job['mode'],projected_gpu_seconds=v['charged']+future,job_cap=job['gpu_seconds'],within_cap=v['charged']+future<=job['gpu_seconds'])
|
||||
|
||||
def project_total(c,estimates,terminal):
|
||||
if any(i not in range(6) for i in estimates) or any(i not in range(6) for i in terminal):raise ValueError('Unknown cell')
|
||||
total=c['postfit_gpu_reserve'];rows=[]
|
||||
for j in c['jobs']:
|
||||
i=j['id'];cost=terminal[i]['gpu_seconds_charged'] if i in terminal else estimates.get(i,{}).get('projected_gpu_seconds',j['prelaunch_projected_gpu_seconds'])
|
||||
if not math.isfinite(cost) or cost<0:raise ValueError('Invalid ledger')
|
||||
rows.append(dict(id=i,projected_gpu_seconds=cost,within_cap=cost<=j['gpu_seconds']));total+=cost
|
||||
return dict(status='PASS' if total<=c['item_gpu_cap'] and all(x['within_cap'] for x in rows) else 'STOP_BUDGET_PROJECTION',rows=rows,projected_item_gpu_seconds=total,postfit_reserve=c['postfit_gpu_reserve'],phase3_cap=c['phase3_gpu_cap'])
|
||||
|
||||
def first_epoch_release(root,jobid):
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('Global item1 stop')
|
||||
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
|
||||
if not p.exists():return False
|
||||
d=read(p)
|
||||
if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('First-epoch projection rejected')
|
||||
return True
|
||||
|
||||
def dispatchable(pending,active_jobs,limit=4):
|
||||
occupied={j['gpu_uuid'] for j in active_jobs};slots=limit-len(active_jobs);selected=[]
|
||||
if slots<0 or len(occupied)!=len(active_jobs):raise ValueError('Active GPU collision/cap')
|
||||
for j in pending:
|
||||
if not slots:break
|
||||
if j['gpu_uuid'] in occupied:continue
|
||||
selected.append(j);occupied.add(j['gpu_uuid']);slots-=1
|
||||
return selected
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Independent bounded ITEM1 fold guard; shared global stop and immutable S1."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase3_data_sufficiency_v2_common import config
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(queue,fold_id):
|
||||
c,_=config(queue);job=c['jobs'][fold_id];c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'fold.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single ITEM1 fold launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds']))
|
||||
def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase3_data_sufficiency_v2_worker.py',str(queue),str(fold_id)]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('ITEM1_FOLD_TIME_LIMIT')
|
||||
if (queue/'global-stop.json').exists():reason('ITEM1_GLOBAL_STOP')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='ITEM1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
try:
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc)))
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('job')!=fold_id):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT')
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'ITEM1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=True,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Single six-cell queue, at most four GPU workers; first-epoch and periodic cost gates; checkpoint-only terminal."""
|
||||
import fcntl,os,signal,subprocess,sys,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase3_data_sufficiency_v2_common import config,read,project_cell,project_total,execution_command,dispatchable
|
||||
from phase2_h1_worker_v2 import save,sha,checked
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from s2_recovery_gpu_supervisor import tree_usage,disk_usage
|
||||
|
||||
def run(root):
|
||||
c,pilot=config(root);validate_review(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single item1 attempt; no retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing item1 execution')
|
||||
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];last_projection={};estimates={};pending=list(c['jobs'])
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True))
|
||||
def charged():return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values())
|
||||
def terminate(v):
|
||||
if v.get('stop_sent') is None:v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic()
|
||||
elif time.monotonic()-v['stop_sent']>20:
|
||||
try:
|
||||
parent=psutil.Process(v['process'].pid)
|
||||
for p in reversed(parent.children(recursive=True)):
|
||||
try:p.kill()
|
||||
except psutil.NoSuchProcess:pass
|
||||
parent.kill()
|
||||
except psutil.NoSuchProcess:pass
|
||||
def collect(i,v):
|
||||
rc=v['process'].poll()
|
||||
if rc is None:return False
|
||||
p=root/'jobs'/str(i)/'complete.json';r=read(p) if p.exists() else None;observed=60+time.monotonic()-v['start'];ok=rc==0 and r is not None and r['status']=='PASS'
|
||||
terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=max(observed,r['gpu_seconds_charged'] if r else 0),guard_complete_sha256=sha(p) if r else None);v['log'].close();del active[i]
|
||||
save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
|
||||
if not ok:stop('CELL_'+str(i)+'_FAILED')
|
||||
return True
|
||||
def observe(j):
|
||||
i=j['id'];rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))];full=[r for r in rows if r['full_epoch']]
|
||||
if not full:return None
|
||||
return dict(steps=rows[-1]['optimizer_steps'],charged=terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-active[i]['start'],max_epoch_seconds=max(r['train_epoch_seconds'] for r in full),max_validation_seconds=max(r['validation_seconds'] for r in rows))
|
||||
def dispatch():
|
||||
for j in dispatchable(pending,[c['jobs'][i] for i in active]):
|
||||
i=j['id'];d=root/'jobs'/str(i);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic();cmd=[sys.executable,'tools/phase3_data_sufficiency_v2_guard.py',str(root),str(i)];p=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,start_new_session=True);active[i]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(pid=p.pid,command=cmd,unix=time.time(),gpu_uuid=j['gpu_uuid']));pending.remove(j)
|
||||
handlers={s:signal.signal(s,lambda n,f:requested.append('SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]}
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),review_sha256=sha(root/'review.json'),new_fits=6,S1_retrained=False,item_gpu_cap=120000,postfit_started=False))
|
||||
try:
|
||||
while active or pending:
|
||||
if not (root/'global-stop.json').exists():dispatch()
|
||||
for i,v in list(active.items()):collect(i,v)
|
||||
for x in requested:stop(x)
|
||||
amount=charged();_,rss,_=tree_usage(os.getpid())
|
||||
if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_TRAIN_GPU_RESERVE_BOUNDARY')
|
||||
if rss>64*2**30 or disk_usage(root)>80*2**30:stop('AGGREGATE_MEMORY_DISK_LIMIT')
|
||||
if not (root/'global-stop.json').exists():
|
||||
for j in c['jobs']:
|
||||
i=j['id']
|
||||
if i not in active:continue
|
||||
obs=observe(j)
|
||||
if obs is None:continue
|
||||
stage=int(obs['steps']/j['target_steps']*20)
|
||||
if stage>last_projection.get(i,-1):
|
||||
cell=project_cell(j,obs);estimates[i]=cell;total=project_total(c,estimates,terminal);p=dict(**total,cell=cell,contract_sha256=sha(root/'contract.json'),unix=time.time())
|
||||
if not cell['within_cap']:p['status']='STOP_BUDGET_PROJECTION'
|
||||
save(root/'jobs'/str(i)/('first-epoch-projection.json' if i not in last_projection else 'latest-cost-projection.json'),p);save(root/'latest-cost-projection.json',p);last_projection[i]=stage
|
||||
if p['status']!='PASS':stop('PROJECTED_ITEM_GPU_BUDGET')
|
||||
if (root/'global-stop.json').exists():
|
||||
for v in active.values():terminate(v)
|
||||
save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'ITEM1_TRAINING',unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],terminal=list(terminal.values()),pending=[j['id'] for j in pending],charged_training_gpu_seconds=amount,postfit_gpu_reserve=c['postfit_gpu_reserve'],item_cap=120000,phase3_cap=612000))
|
||||
if (root/'global-stop.json').exists() and not active:break
|
||||
if active:time.sleep(1)
|
||||
if (root/'global-stop.json').exists() or len(terminal)!=6 or any(x['status']!='PASS' for x in terminal.values()):raise RuntimeError('Item1 train incomplete')
|
||||
config(root);models=[]
|
||||
for j in c['jobs']:
|
||||
w=root/'jobs'/str(j['id'])/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256']))
|
||||
if r['optimizer_steps']!=j['target_steps'] or stats['parts']!=c['pools'][str(j['pool'])]['learning'] or r['test_targets_read']:raise ValueError('Terminal cell identity')
|
||||
for m in r['checkpoints']:checked(m['path'],m['sha256']);models.append(dict(job=j['id'],pool=j['pool'],mode=j['mode'],**m))
|
||||
if len(models)!=6 or charged()>c['train_gpu_cap']:raise ValueError('Checkpoint or cumulative cost')
|
||||
save(root/'models-terminal.json',dict(models=models,reused_S1=c['S1_reuse'],new_fits=6,new_checkpoints=6,test_prediction_started=False))
|
||||
freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]])
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',entries=list(terminal.values()),training_gpu_seconds=charged(),postfit_gpu_reserve=c['postfit_gpu_reserve'],item_gpu_cap=120000,coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,no_retry=True,test_targets_read=False));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json']);return 0
|
||||
except BaseException as exc:
|
||||
stop(type(exc).__name__+': '+str(exc))
|
||||
while active:
|
||||
for i,v in list(active.items()):
|
||||
try:
|
||||
if not collect(i,v):terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:terminal[i]=dict(id=i,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start']);v['log'].close();del active[i]
|
||||
if active:time.sleep(.5)
|
||||
save(root/'complete.json',dict(status='STOPPED',reason=str(exc),entries=list(terminal.values()),never_attempted=[j['id'] for j in pending],training_gpu_seconds=charged(),postfit_started=False,no_retry=True));return 1
|
||||
finally:
|
||||
for s,h in handlers.items():signal.signal(s,h)
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Six fixed fresh fits; each update budget has a complete schedule; no test prediction/target access."""
|
||||
import os,sys,time,traceback,collections
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase3_data_sufficiency_v2_common import config,view_for,lr_step,first_epoch_release
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_data import standardization,load_part,to_device,arguments,cached
|
||||
from phase2_h2_pilot_worker import seed,peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,jobid):
|
||||
c,pilot=config(root);j=c['jobs'][jobid];view=view_for(c,pilot,j);f=view['folds'][0];out=root/'jobs'/str(jobid)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,j['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();steps=0;epoch=0;stage='SETUP';exposures=collections.Counter();nodes_exposed=0;edges_exposed=0;checkpoints=[];val_checks=0
|
||||
def record(extra,final=False):
|
||||
d=dict(stage=stage,job=jobid,pool=j['pool'],mode=j['mode'],epoch=epoch,optimizer_steps=steps,nodes_exposed=nodes_exposed,edges_exposed=edges_exposed,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**extra,**peaks());save(out/'progress.json',d)
|
||||
if final:save(out/'history'/f'{epoch:04d}.json',d)
|
||||
return d
|
||||
def checkpoint(tag):
|
||||
path=out/(tag+'.pt');torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,pool=j['pool'],mode=j['mode'],contract_sha256=sha(root/'contract.json'),role=tag),path)
|
||||
checkpoints.append(dict(tag=tag,steps=steps,path=str(path),sha256=sha(path),standardization=str(out/'standardization.json'),standardization_sha256=sha(out/'standardization.json')))
|
||||
def validation():
|
||||
nonlocal val_checks
|
||||
begin=time.monotonic();model.eval();loss=0.;n=0
|
||||
with torch.no_grad():
|
||||
for case in f['validation']:
|
||||
d=to_device(load_part(view,case,stats,'validation',out),'cuda');p=model(*arguments(d),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as log:
|
||||
import json
|
||||
log.write(json.dumps(dict(case=case,steps=steps,role='validation_diagnostic_only',target_sha256=view['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(view,case,'target'),d['F'],f['c'],fit_dtype='float32');delta=p.cpu().numpy().astype(np.float64)-target;loss+=float(delta@delta);n+=d['nodes'];del d,p
|
||||
torch.cuda.synchronize()
|
||||
if n!=f['validation_nodes']:raise ValueError('Validation coverage')
|
||||
val_checks+=1;v=dict(steps=steps,validation_mse=loss/n,validation_nodes=n,validation_seconds=time.monotonic()-begin,validation_used_for_selection=False);save(out/f'validation-{steps:06d}.json',v);return v
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=j['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
||||
stats=standardization(view,out)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed();model=H2Geo().cuda();opt=optimizer(model)
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=j['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),seed=0,fresh_initialization=True,model_dtype='FP32',autocast=False))
|
||||
while steps<j['target_steps']:
|
||||
epoch+=1;stage='TRAIN';begin=time.monotonic();seen=0;enodes=0;loss_sum=0.;validation_seconds=0.
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(len(f['learning']))
|
||||
for index in order:
|
||||
case=f['learning'][int(index)];lr=lr_epoch(epoch) if j['mode']=='epoch' else lr_step(steps+1,total=j['target_steps'])
|
||||
for group in opt.param_groups:group['lr']=lr
|
||||
d=to_device(load_part(view,case,stats,'learning',out),'cuda');res=train_step(model,opt,d,len(f['learning']),f['learning_nodes']);steps+=1;seen+=1;enodes+=d['nodes'];nodes_exposed+=d['nodes'];edges_exposed+=d['directed_edges'];exposures[case]+=1;loss_sum+=res['loss'];del d
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=enodes,lr=lr,loss=res['loss']))
|
||||
if j['mode']=='step' and steps==j['target_steps']:checkpoint('U' if steps==c['U'] else '2U')
|
||||
if j['mode']=='step' and steps%3200==0:validation_seconds+=validation()['validation_seconds']
|
||||
if steps==j['target_steps']:break
|
||||
train_seconds=time.monotonic()-begin-validation_seconds
|
||||
if j['mode']=='epoch' and epoch%10==0:validation_seconds+=validation()['validation_seconds']
|
||||
full_epoch=seen==len(f['learning'])
|
||||
if full_epoch and enodes!=f['learning_nodes']:raise ValueError('Epoch full-node identity')
|
||||
stage='EPOCH_COMPLETE';record(dict(epoch_parts=seen,epoch_nodes=enodes,full_epoch=full_epoch,train_epoch_seconds=train_seconds,validation_seconds=validation_seconds,train_online_mean_weighted_loss=loss_sum/seen,not_frozen_model_training_domain_error=True),final=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_FIRST_EPOCH_PROJECTION'
|
||||
while not first_epoch_release(root,jobid):
|
||||
record(dict(waiting=True));time.sleep(1)
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('Global item1 stop')
|
||||
if j['mode']=='epoch':checkpoint('epoch200')
|
||||
expected=['epoch200'] if j['mode']=='epoch' else ['U' if j['target_steps']==c['U'] else '2U']
|
||||
if [x['tag'] for x in checkpoints]!=expected:raise ValueError('Checkpoint coverage')
|
||||
save(out/'exposures.json',dict(per_part=dict(exposures),nodes_exposed=nodes_exposed,edges_exposed=edges_exposed,steps=steps,common_steps_not_equal_compute=True))
|
||||
save(out/'result.json',dict(status='COMPLETE',job=jobid,pool=j['pool'],mode=j['mode'],completed_epochs=epoch,optimizer_steps=steps,checkpoints=checkpoints,validation_checks=val_checks,standardization_sha256=sha(out/'standardization.json'),exposures_sha256=sha(out/'exposures.json'),test_targets_read=False,test_prediction_started=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
save(out/'failure.json',dict(status='FAILED',job=jobid,stage=stage,epoch=epoch,optimizer_steps=steps,reason=str(exc),traceback=traceback.format_exc(),**(peaks() if torch.cuda.is_initialized() else {})));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Metadata-only preparation of Claude8643/8710 amendment, preserving unrun v1."""
|
||||
import json,time,shutil,copy
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase2_h2_prepare import closure
|
||||
from phase2_home_freeze_io import verify,freeze
|
||||
from phase3_data_sufficiency_v2_common import config,execution_command
|
||||
B=Path('/home/frank14f/optane/MeltMind');R=Path('/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18');O=R.parent/'phase3-data-sufficiency-2026-11-18'
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def main():
|
||||
import psutil
|
||||
start=time.process_time()
|
||||
if R.exists() or (O/'started.json').exists():raise FileExistsError('Existing newroot or started oldroot')
|
||||
if any(execution_command(p.info['cmdline'] or []) for p in psutil.process_iter(['cmdline'])):raise RuntimeError('Existing v2 task')
|
||||
verify(O,'implementation');old=read(O/'contract.json');c=copy.deepcopy(old)
|
||||
for p,h in old['source_sha256'].items():checked(p,h)
|
||||
p0=read(O/'prelaunch-projection.json');t=p0['source_max_train_epoch_seconds'];v=p0['source_max_validation_seconds'];jobs=[];projections=[]
|
||||
for i,(pool,mode,steps,cap) in enumerate([(25,'epoch',22200,11000),(50,'epoch',38000,17000),(50,'step',32000,14000),(50,'step',64000,28400),(100,'step',32000,14000),(100,'step',64000,28400)]):
|
||||
f=c['pools'][str(pool)];n=len(f['learning']);f0=c['pools']['100'];ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],n/320);checks=20 if mode=='epoch' else steps//3200;est=1.25*(t*ratio/n*steps+checks*v)+180;j=copy.deepcopy(old['jobs'][i%4]);j.update(id=i,pool=pool,mode=mode,step_budget=steps if mode=='step' else None,target_steps=steps,learning_parts=n,learning_nodes=f['learning_nodes'],learning_edges=f['learning_edges'],gpu_seconds=cap,cpu_seconds=cap,prelaunch_projected_gpu_seconds=est);jobs.append(j);projections.append(dict(id=i,pool=pool,mode=mode,steps=steps,expected_new_gpu_seconds=est,job_cap=cap,within_cap=est<=cap,validation_checks=checks))
|
||||
total=sum(x['expected_new_gpu_seconds'] for x in projections)+7200
|
||||
assert all(x['within_cap'] for x in projections) and total<=120000
|
||||
c.update(version='phase3-item1-training-v2',approval='1794948643.973609',budget_approval='1794948710.366709',new_fits=6,jobs=jobs,item_gpu_cap=120000,approved_item_gpu_cap=120000,train_gpu_cap=112800,postfit_gpu_reserve=7200,supersedes_unstarted_root=str(O),supersedes_implementation='c8e2ee4ca3d08cf0080ac18a1d17bfb42fb1b8a6',schedule='Each step fit freshly initialized; warmup3200 then cosine to its own U/2U endpoint',interpretation='Update-budget effect includes complete schedule change; not pure common-trajectory continuation')
|
||||
c['source_sha256'][str(O/'contract.json')]=sha(O/'contract.json');c['source_sha256'][str(O/'implementation-git-freeze.json')]=sha(O/'implementation-git-freeze.json')
|
||||
comp=read(O/'comparison-contract.json');comp.update(revision='Claude1794948643 complete-schedule amendment to e24fff7c',schedule='Four separate fresh step-budget fits, own complete warmup3200+cosine endpoint',update_effect='joint update budget plus complete LR schedule, not pure continuation',new_fits=6,old_entry_never_started=True)
|
||||
R.mkdir();save(R/'comparison-contract.json',comp);c['comparison_contract']=str(R/'comparison-contract.json');c['comparison_contract_sha256']=sha(R/'comparison-contract.json')
|
||||
codes={str(B/p):d for p,d in closure(['phase3_data_sufficiency_v2_common','phase3_data_sufficiency_v2_worker','phase3_data_sufficiency_v2_guard','phase3_data_sufficiency_v2_queue','prepare_phase3_data_sufficiency_v2','test_phase3_data_sufficiency_v2']).items()};doc=B/'docs/phase3-data-sufficiency-v2-entry-2026-11-18.md';codes[str(doc)]=sha(doc);c['code_sha256']=codes;save(R/'contract.json',c)
|
||||
shutil.copyfile('/tmp/phase3-data-sufficiency-v2-tests1.log',R/'synthetic-attempt1.log');assert (R/'synthetic-attempt1.log').read_text().rstrip().endswith('OK');save(R/'acceptance.json',dict(status='PASS',tests=16,scope='CPU schedule/dispatch/GPU ownership/partial traversal/cost/no-double-count/stop-gate checks',test_targets_read=False,GPU_workload=False))
|
||||
save(R/'prelaunch-projection.json',dict(cells=projections,total_gpu_seconds=total,postfit_reserve=7200,item_cap=120000,phase3_cap=612000,source_max_epoch=t,source_max_validation=v,safety_factor=1.25,actual_run=False))
|
||||
save(R/'amendment.json',dict(science_ruling='1794948643.973609',budget_ruling='1794948710.366709',old_entry=str(O),old_implementation=c['supersedes_implementation'],old_started=False,old_common_trajectory_preserved=True,new_complete_schedules=True,new_fits=6,original_e24_spec_not_unchanged=True))
|
||||
save(R/'preparation-audit.json',dict(status='PREPARED_NOT_STARTED',metadata_cpu_seconds=time.process_time()-start,source_pins_verified=True,old_root_unstarted=True,no_target_arrays=True,no_GPU_workload=True))
|
||||
freeze(R,'implementation',[R/'contract.json',R/'comparison-contract.json',R/'acceptance.json',R/'prelaunch-projection.json',R/'amendment.json',R/'preparation-audit.json',R/'synthetic-attempt1.log',*[Path(p) for p in codes]])
|
||||
binding=dict(reviewed_commit=verify(R,'implementation')['commit'],contract_sha256=sha(R/'contract.json'),implementation_proof_sha256=sha(R/'implementation-git-freeze.json'),comparison_contract_sha256=c['comparison_contract_sha256'],code_sha256=codes);save(R/'expected-review-binding.json',binding);freeze(R,'review-binding',[R/'contract.json',R/'comparison-contract.json',R/'expected-review-binding.json',R/'implementation-git-freeze.json']);config(R)
|
||||
print(json.dumps(dict(root=str(R),implementation=binding['reviewed_commit'],contract_sha256=binding['contract_sha256'],binding_sha256=sha(R/'expected-review-binding.json'),projected_gpu_seconds=total,new_fits=6,started=False)))
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,35 @@
|
||||
import unittest,tempfile
|
||||
from pathlib import Path
|
||||
from phase3_data_sufficiency_v2_common import lr_step,project_cell,project_total,first_epoch_release,dispatchable
|
||||
class V2Tests(unittest.TestCase):
|
||||
def test_complete_schedule_both_budgets(self):
|
||||
for end in [32000,64000]:
|
||||
self.assertAlmostEqual(lr_step(1,end),1e-4);self.assertAlmostEqual(lr_step(3200,end),1e-3);self.assertAlmostEqual(lr_step(end,end),1e-5)
|
||||
def test_distinct_U_schedule(self):self.assertNotEqual(lr_step(32000,32000),lr_step(32000,64000))
|
||||
def test_common_warmup(self):self.assertEqual(lr_step(1600,32000),lr_step(1600,64000))
|
||||
def test_partial_epoch(self):self.assertEqual(divmod(32000,190),(168,80));self.assertEqual(divmod(64000,190),(336,160))
|
||||
def test_invalid_step(self):
|
||||
for end in [32000,64000]:
|
||||
with self.assertRaises(ValueError):lr_step(end+1,end)
|
||||
def jobs(self):return [dict(id=i,pool=50,mode='step',target_steps=32000,learning_parts=190,validation_seconds_bound=10,gpu_seconds=14000,prelaunch_projected_gpu_seconds=12000,gpu_uuid=str(i%4)) for i in range(6)]
|
||||
def c(self):return dict(jobs=self.jobs(),postfit_gpu_reserve=7200,item_gpu_cap=120000,phase3_gpu_cap=612000)
|
||||
def test_initial_four_dispatch(self):self.assertEqual([j['id'] for j in dispatchable(self.jobs(),[])],[0,1,2,3])
|
||||
def test_gpu_ownership(self):self.assertEqual([j['id'] for j in dispatchable(self.jobs()[4:],[self.jobs()[1],self.jobs()[2],self.jobs()[3]])],[4])
|
||||
def test_no_duplicate_gpu(self):
|
||||
with self.assertRaises(ValueError):dispatchable([], [self.jobs()[0],self.jobs()[4]])
|
||||
def test_no_more_than_four(self):self.assertEqual(dispatchable(self.jobs()[4:],self.jobs()[:4]),[])
|
||||
def test_pending_projection_charged(self):self.assertEqual(project_total(self.c(),{}, {})['projected_item_gpu_seconds'],79200)
|
||||
def test_terminal_not_double_counted(self):self.assertEqual(project_total(self.c(),{0:dict(projected_gpu_seconds=13500)},{0:dict(gpu_seconds_charged=10000)})['projected_item_gpu_seconds'],77200)
|
||||
def test_cell_cap(self):
|
||||
p=project_cell(self.jobs()[0],dict(steps=190,max_epoch_seconds=200,max_validation_seconds=0,charged=100));self.assertFalse(p['within_cap'])
|
||||
def test_total_cap(self):
|
||||
c=self.c();c['item_gpu_cap']=79000;self.assertEqual(project_total(c,{}, {})['status'],'STOP_BUDGET_PROJECTION')
|
||||
def test_nan_cost(self):
|
||||
with self.assertRaises(ValueError):project_cell(self.jobs()[0],dict(steps=190,max_epoch_seconds=float('nan'),max_validation_seconds=0,charged=100))
|
||||
def test_cell_specific_gate(self):
|
||||
with tempfile.TemporaryDirectory() as p:self.assertFalse(first_epoch_release(Path(p),4))
|
||||
def test_global_stop(self):
|
||||
with tempfile.TemporaryDirectory() as p:
|
||||
r=Path(p);(r/'global-stop.json').write_text('{}')
|
||||
with self.assertRaises(RuntimeError):first_epoch_release(r,0)
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user