Preserve item1 terminal results and freeze seed1 training review entry
This commit is contained in:
1 parent
888f706612
commit
c0f89a5e95
114 files changed
+282280
No files matched your search
@@ -0,0 +1,100 @@
|
||||
"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked,sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_h2_common import read
|
||||
|
||||
GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
||||
|
||||
|
||||
def execution_command(args):
|
||||
return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py'} for x in args)
|
||||
|
||||
|
||||
def fold_view(pilot,fold):
|
||||
if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds')
|
||||
f=pilot['folds'][fold]
|
||||
if f['fold']!=fold:raise ValueError('Actual original fold identity')
|
||||
roles=[set(f[k]) for k in ('learning','validation','test')]
|
||||
if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap')
|
||||
return dict(pilot,folds=[f])
|
||||
|
||||
|
||||
def config(root):
|
||||
c=read(root/'contract.json');verify(root,'implementation')
|
||||
if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope')
|
||||
if c['dispatch_approval']!='1794983024.445549' or c['approval']!='1794982860.347839' or c['gpu_cap']!=118800 or c['train_gpu_cap']!=111600 or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000:raise PermissionError('Frozen seed1 and phase3 ceilings')
|
||||
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
||||
pilot=read(c['pilot_contract'])
|
||||
import numpy as np,torch
|
||||
if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions')
|
||||
if sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Partitioned training budget')
|
||||
if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve')
|
||||
tests=[]
|
||||
for j in c['jobs']:
|
||||
f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test'])
|
||||
for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']:
|
||||
if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k)
|
||||
if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved GPU assignment and all epochs')
|
||||
if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition')
|
||||
return c,pilot
|
||||
|
||||
|
||||
def project_cell(job,observation):
|
||||
e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged']
|
||||
validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0))
|
||||
if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing')
|
||||
future=1.25*((200-e)*train+(20-e//10)*validation)+120
|
||||
value=charged+future
|
||||
return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,within_cap=value<=job['gpu_seconds'])
|
||||
|
||||
|
||||
def project_total(c,estimates,terminal,active_charges=None):
|
||||
active_charges=active_charges or {};ids={j['id'] for j in c['jobs']}
|
||||
if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job')
|
||||
rows=[];total=c['postfit_gpu_reserve']
|
||||
for j in c['jobs']:
|
||||
i=j['id'];v=terminal[i]['gpu_seconds_charged'] if i in terminal else max(active_charges.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',j['gpu_seconds']))
|
||||
if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger')
|
||||
total+=v;rows.append(dict(id=i,projected_gpu_seconds=v,within_cap=v<=j['gpu_seconds']))
|
||||
ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['within_cap'] for r in rows)
|
||||
return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve'])
|
||||
|
||||
|
||||
def first_epoch_release(root,jobid):
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop')
|
||||
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
|
||||
if not p.exists():return False
|
||||
r=read(p)
|
||||
if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected')
|
||||
return True
|
||||
|
||||
|
||||
def dispatchable(pending,active,available):
|
||||
occupied={j['gpu_uuid'] for j in active}
|
||||
if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU')
|
||||
free=[u for u in GPUS if u in available and u not in occupied]
|
||||
return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)]
|
||||
|
||||
|
||||
def assigned_job(root,c,index,digest):
|
||||
p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p)
|
||||
if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity')
|
||||
return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid'])
|
||||
|
||||
|
||||
def available_gpus(c):
|
||||
import psutil
|
||||
available={GPUS[0]}
|
||||
item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root'])
|
||||
names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []}
|
||||
if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}:
|
||||
available.add(GPUS[1])
|
||||
if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}:
|
||||
# GPU2 stays reserved for the already-prioritized Track A postfit until its terminal.
|
||||
post=Path(c['track_a_postfit_root'])
|
||||
available.add(GPUS[3])
|
||||
if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}:
|
||||
available.add(GPUS[2])
|
||||
return available
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Independent bounded SEED1 fold guard; shared global stop and immutable S1."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase3_seed1_common import config,assigned_job
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(queue,fold_id,assignment_sha):
|
||||
c,_=config(queue);job=assigned_job(queue,c,fold_id,assignment_sha);c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-seed1-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single SEED1 fold launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds']))
|
||||
def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase3_seed1_worker.py',str(queue),str(fold_id),assignment_sha]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('SEED1_FOLD_TIME_LIMIT')
|
||||
if (queue/'global-stop.json').exists():reason('SEED1_GLOBAL_STOP')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='SEED1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
try:
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc)))
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('fold')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT')
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'SEED1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,seed1_started=True,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Five original grouped folds on released GPUs; train and freeze only, never score."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
||||
from phase3_seed1_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus
|
||||
|
||||
|
||||
def run(root):
|
||||
c, _ = config(root)
|
||||
validate_review(root)
|
||||
lock = (root / 'queue.lock').open('a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'started.json').exists():
|
||||
raise FileExistsError('One Seed1 attempt; no retry')
|
||||
for p in psutil.process_iter(['pid', 'cmdline']):
|
||||
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
|
||||
raise RuntimeError('Existing Seed1 execution')
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
||||
pending = list(c['jobs'])
|
||||
requested = []
|
||||
|
||||
def stop(reason):
|
||||
if not (root / 'global-stop.json').exists():
|
||||
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
|
||||
|
||||
def active_charges():
|
||||
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
for i, v in active.items()}
|
||||
|
||||
def charged():
|
||||
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
|
||||
|
||||
def terminate(v):
|
||||
if v.get('stop_sent') is None:
|
||||
if v['process'].poll() is None:
|
||||
v['process'].send_signal(signal.SIGTERM)
|
||||
v['stop_sent'] = time.monotonic()
|
||||
elif time.monotonic() - v['stop_sent'] > 20:
|
||||
try:
|
||||
parent = psutil.Process(v['process'].pid)
|
||||
for p in reversed(parent.children(recursive=True)):
|
||||
try:
|
||||
p.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
parent.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def collect(i, v):
|
||||
rc = v['process'].poll()
|
||||
if rc is None:
|
||||
return False
|
||||
p = root / 'jobs' / str(i) / 'complete.json'
|
||||
r = read(p) if p.exists() else None
|
||||
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
|
||||
and not r.get('monitor_errors') and r.get('worker') is not None
|
||||
and not r['worker'].get('monitor_errors'))
|
||||
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
|
||||
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
|
||||
guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json'))
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
|
||||
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
|
||||
if not ok:
|
||||
stop('JOB_' + str(i) + '_FAILED')
|
||||
return True
|
||||
|
||||
def dispatch():
|
||||
for j in dispatchable(pending, [v['job'] for v in active.values()], available_gpus(c)):
|
||||
i = j['id']
|
||||
d = root / 'jobs' / str(i)
|
||||
d.mkdir(parents=True, exist_ok=False)
|
||||
log = (d / 'entry.log').open('x')
|
||||
begin = time.monotonic()
|
||||
save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(available_gpus(c))))
|
||||
cmd = [sys.executable, 'tools/phase3_seed1_guard.py', str(root), str(i), sha(d/'assignment.json')]
|
||||
try:
|
||||
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
except BaseException:
|
||||
log.close()
|
||||
raise
|
||||
active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j)
|
||||
pending.remove(c['jobs'][i])
|
||||
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
|
||||
|
||||
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
|
||||
for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
|
||||
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
|
||||
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
|
||||
try:
|
||||
while active or pending:
|
||||
for i, v in list(active.items()):
|
||||
collect(i, v)
|
||||
for reason in requested:
|
||||
stop(reason)
|
||||
amount = charged()
|
||||
_, rss, _ = tree_usage(os.getpid())
|
||||
if amount >= c['train_gpu_cap'] - 80:
|
||||
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
|
||||
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
|
||||
stop('AGGREGATE_MEMORY_DISK_LIMIT')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
for i in active:
|
||||
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
|
||||
rows = [read(p) for p in files]
|
||||
if not rows:
|
||||
continue
|
||||
epoch = rows[-1]['epoch']
|
||||
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
||||
if epoch > last_epoch.get(i, 0):
|
||||
observation = dict(epoch=epoch, charged=active_charges()[i],
|
||||
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
|
||||
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
|
||||
cell = project_cell(c['jobs'][i], observation)
|
||||
estimates[i] = cell
|
||||
projection = dict(**project_total(c, estimates, terminal, active_charges()),
|
||||
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
|
||||
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
||||
save(root / 'jobs' / str(i) / name, projection)
|
||||
save(root / 'latest-cost-projection.json', projection)
|
||||
last_epoch[i] = epoch
|
||||
if projection['status'] != 'PASS':
|
||||
stop('PROJECTED_SEED1_BUDGET')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
dispatch()
|
||||
if (root / 'global-stop.json').exists():
|
||||
for v in active.values():
|
||||
terminate(v)
|
||||
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
||||
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
||||
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
|
||||
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
|
||||
if (root / 'global-stop.json').exists() and not active:
|
||||
break
|
||||
if active or pending:
|
||||
time.sleep(1)
|
||||
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
|
||||
raise RuntimeError('Seed1 training incomplete')
|
||||
config(root)
|
||||
models = []
|
||||
for j in c['jobs']:
|
||||
w = root / 'jobs' / str(j['id']) / 'worker'
|
||||
r = read(w / 'result.json')
|
||||
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
|
||||
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
|
||||
or stats['parts'] != j['learning'] or r['test_targets_read']):
|
||||
raise ValueError('Terminal model identity')
|
||||
checked(r['checkpoint'], r['checkpoint_sha256'])
|
||||
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
|
||||
standardization=str(w / 'standardization.json'),
|
||||
standardization_sha256=r['standardization_sha256']))
|
||||
if charged() > c['train_gpu_cap']:
|
||||
raise ValueError('Terminal cumulative cost')
|
||||
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
|
||||
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
|
||||
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
|
||||
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
|
||||
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
|
||||
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
|
||||
phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(),
|
||||
no_retry=True, test_targets_read=False, postfit_started=False))
|
||||
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
|
||||
return 0
|
||||
except BaseException as exc:
|
||||
stop(type(exc).__name__ + ': ' + str(exc))
|
||||
while active:
|
||||
for i, v in list(active.items()):
|
||||
try:
|
||||
if not collect(i, v):
|
||||
terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:
|
||||
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
||||
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
if active:
|
||||
time.sleep(.5)
|
||||
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
|
||||
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
|
||||
postfit_started=False, no_retry=True))
|
||||
return 1
|
||||
finally:
|
||||
for s, handler in handlers.items():
|
||||
signal.signal(s, handler)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,67 @@
|
||||
"""One new S2 fold, same frozen recipe with seed1/200epochs; no test prediction or target reads."""
|
||||
import json,os,sys,time,traceback,random
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from phase3_seed1_common import config,fold_view,first_epoch_release,assigned_job
|
||||
from phase2_h2_data import standardization
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id,assignment_sha):
|
||||
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(1);np.random.seed(1);torch.manual_seed(1);torch.cuda.manual_seed_all(1)
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=1,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([1,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=1,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Metadata-only seed1 five-fold preparation; immutable seed0 and Track A sources."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase2_h2_s2_common import config as original_config
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from track_a_h2_common import read
|
||||
|
||||
B=Path(__file__).resolve().parents[1];H=Path('/home/frank14f/MeltMind-runs');R=H/'phase3-seed1-2026-11-18'
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if R.exists():raise FileExistsError('Existing seed1 root; no duplicate')
|
||||
old=H/'phase2-h2-s2-2026-11-17';post=H/'phase2-h2-s2-postfit-2026-11-17';item=H/'phase3-item1-postfit-recovery-2026-11-18'
|
||||
oc,pilot=original_config(old);verify(post,'terminal');verify(item,'terminal')
|
||||
pd=read(post/'complete.json');idone=read(item/'complete.json');cost=read(item/'post-terminal-cost.json')
|
||||
if pd['decision']!='CONFIRMED' or pd['status']!='COMPLETE' or idone['status']!='COMPLETE' or not cost['within_cap']:raise PermissionError('Prior approved stage terminal required')
|
||||
source=dict(oc['source_sha256'])
|
||||
for p,h in oc['code_sha256'].items():checked(p,h);source[p]=h
|
||||
spec=B/'references/2026-11-18/phase3-seed-sensitivity/mold-seed-ensemble-spec.md';checked(spec,'5dce5d0feb2b5cdd2703f53f0030b8127915d3464c7fa37e16507303b9c52b0d')
|
||||
for p in [old/'contract.json',old/'complete.json',post/'complete.json',post/'post-terminal-cost.json',post/'score/result.json',post/'predictions-manifest.json',post/'contract.json',item/'complete.json',item/'post-terminal-cost.json',item/'terminal-git-freeze.json',item/'terminal-authorization.json',spec]:source[str(p)]=sha(p)
|
||||
jobs=[];timings=[]
|
||||
for i,f in enumerate(pilot['folds']):
|
||||
hist=B/'artifacts/phase2-h2-s1-2026-11-17/worker/history' if i==0 else old/'folds'/str(i)/'worker/history'
|
||||
records=[hist/f'{e:03d}.json' for e in range(10,201,10)]
|
||||
for p in records:source[str(p)]=sha(p)
|
||||
val=max(read(p)['validation_seconds'] for p in records)
|
||||
jobs.append(dict(id=i,fold=i,learning=f['learning'],validation=f['validation'],test=f['test'],learning_nodes=f['learning_nodes'],validation_nodes=f['validation_nodes'],clock_path=f['clock_path'],clock_sha256=f['clock_sha256'],c=f['c'],target_steps=200*len(f['learning']),learning_parts=len(f['learning']),worker_cpus=[2*i,2*i+1],guard_cpus=[10+i],gpu_seconds=22320,cpu_seconds=22320,setup_gpu_seconds=60,validation_seconds_bound=val,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30))
|
||||
timings.append(dict(fold=i,validation_seconds_bound=val,source_files=[str(p) for p in records]))
|
||||
codes=local_code_closure([B/'tools'/n for n in ['phase3_seed1_common.py','phase3_seed1_worker.py','phase3_seed1_guard.py','phase3_seed1_queue.py','prepare_phase3_seed1.py','test_phase3_seed1.py']])
|
||||
# Include the original scientific closure, including package files not resolved by local AST imports.
|
||||
for p,h in oc['code_sha256'].items():
|
||||
if p not in codes:codes[p]=h
|
||||
for p,h in source.items():checked(p,h)
|
||||
R.mkdir()
|
||||
comparison=dict(version='seed1-training-comparison-v1',source_spec_sha256=sha(spec),scope='five fresh seed1 grouped folds, original seed0 reused only in later separately reviewed postfit',folds=pilot['folds'],seed=1,epochs=200,selection='last_epoch_only',validation='diagnostic only, original every10epochs',metric='unchanged S2 query readout in later postfit',seed2=False,ensemble=False,original_S2_score_sha256=sha(post/'score/result.json'),original_S2_predictions_sha256=sha(post/'predictions-manifest.json'))
|
||||
save(R/'comparison-contract.json',comparison)
|
||||
c=dict(version='phase3-seed1-v1',approval='1794982860.347839',dispatch_approval='1794983024.445549',seed=1,epochs=200,new_fits=5,gpu_cap=118800,train_gpu_cap=111600,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,phase3_gpu_cap=612000,phase3_prior_gpu_once=cost['cumulative_item_gpu_seconds'],item1_recovery_root=str(item),track_a_training_root=str(H/'track-a-h2-2026-11-18'),track_a_postfit_root=str(H/'track-a-postfit-2026-11-18'),pilot_contract=oc['pilot_contract'],jobs=jobs,coordinator_cpus=[14,15],aggregate_rss_bytes=64*2**30,aggregate_disk_bytes=100*2**30,source_sha256=source,code_sha256=codes,comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(R/'comparison-contract.json'),dispatch='next_pending_fold_first_allowed_free_gpu; GPU2 reserved until Track A postfit terminal',no_retry=True,no_cross_card_retry=True,postfit_started=False,stop_after='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING',new_seed0_fits=0,new_seed2_fits=0,ensemble_started=False)
|
||||
save(R/'contract.json',c);save(R/'preparation-audit.json',dict(status='PREPARED_PENDING_LIGHT_DIFF_REVIEW',code_files=len(codes),contract_sha256=sha(R/'contract.json'),timing_sources=timings,metadata_cpu_seconds=time.process_time()-start,source_pins='PASS',scientific_arrays_read=False,new_model_weights_read=False,test_targets_read=False,new_fits_launched=False,phase3_item1_gpu_once=cost['cumulative_item_gpu_seconds']))
|
||||
print(read(R/'preparation-audit.json')|{'timing_sources':'five original per-fold validation timing maxima'})
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,91 @@
|
||||
"""CPU synthetic seed/role/cost/release/guard tests; no real arrays or GPU."""
|
||||
import ast,copy,inspect,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
from phase3_seed1_common import fold_view,project_cell,project_total,dispatchable,first_epoch_release,available_gpus,GPUS,assigned_job
|
||||
from phase2_h2_data import load_part,standardization
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
import phase2_h2_s2_worker as original
|
||||
import phase3_seed1_worker as new
|
||||
import phase3_seed1_guard as guard
|
||||
|
||||
class SeedTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.pilot=dict(folds=[dict(fold=i,learning=['l'+str(i)],validation=['v'+str(i)],test=['t'+str(i)],learning_nodes=2) for i in range(5)],parts={'l'+str(i):dict(nodes=2) for i in range(5)})
|
||||
self.jobs=[dict(id=i,gpu_seconds=22320,validation_seconds_bound=20) for i in range(5)]
|
||||
self.c=dict(jobs=self.jobs,gpu_cap=118800,postfit_gpu_reserve=7200,phase3_gpu_cap=612000,phase3_prior_gpu_once=73000)
|
||||
def test_all_five_original_roles(self):
|
||||
old=copy.deepcopy(self.pilot)
|
||||
for i in range(5):self.assertEqual(fold_view(self.pilot,i)['folds'][0],self.pilot['folds'][i])
|
||||
self.assertEqual(old,self.pilot)
|
||||
def test_foreign_and_overlap_rejected(self):
|
||||
for i in [True,-1,5]:
|
||||
with self.assertRaises(PermissionError):fold_view(self.pilot,i)
|
||||
self.pilot['folds'][0]['test']=['l0']
|
||||
with self.assertRaises(ValueError):fold_view(self.pilot,0)
|
||||
def test_test_role_never_used_for_train_or_normalization(self):
|
||||
for i in range(5):
|
||||
v=fold_view(self.pilot,i)
|
||||
for role in ['learning','validation']:
|
||||
with self.assertRaises(PermissionError):load_part(v,'t'+str(i),{},role,Path('/unused'))
|
||||
def features(c,case,name):
|
||||
self.assertEqual((case,name),('l'+str(i),'features'));return np.vstack([np.zeros(19),np.full(19,2.)])
|
||||
with tempfile.TemporaryDirectory() as d,patch('phase2_h2_data.cached',features):
|
||||
s=standardization(v,Path(d));self.assertEqual(s['parts'],['l'+str(i)]);self.assertEqual(s['mean'],[1.]*19)
|
||||
def test_epoch_math_identical_except_seed_and_cost_wait(self):
|
||||
def loop(fn):
|
||||
t=ast.parse(inspect.getsource(fn));n=next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch')
|
||||
n.body=[b for b in n.body if not(isinstance(b,ast.If) and ast.unparse(b.test)=='epoch == 1')]
|
||||
return n
|
||||
a,b=loop(original.run),loop(new.run)
|
||||
seq=[n for n in ast.walk(b) if isinstance(n,ast.Call) and ast.unparse(n.func)=='np.random.SeedSequence'];self.assertEqual(len(seq),1);self.assertEqual(seq[0].args[0].elts[0].value,1);seq[0].args[0].elts[0].value=0
|
||||
self.assertEqual(ast.dump(a),ast.dump(b))
|
||||
def test_all_rng_seed1_and_final_checkpoint(self):
|
||||
tree=ast.parse(inspect.getsource(new.run));calls={ast.unparse(n.func):n for n in ast.walk(tree) if isinstance(n,ast.Call)}
|
||||
for name in ['random.seed','np.random.seed','torch.manual_seed','torch.cuda.manual_seed_all']:self.assertEqual(calls[name].args[0].value,1)
|
||||
saves=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and ast.unparse(n.func)=='torch.save'];self.assertEqual(len(saves),1)
|
||||
fields={k.arg:ast.literal_eval(k.value) for k in saves[0].args[0].keywords if k.arg in ['seed','role']};self.assertEqual(fields,dict(seed=1,role='last_epoch_only'))
|
||||
def test_first_epoch_gate_not_all_folds_barrier(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);(p/'jobs/0').mkdir(parents=True);save(p/'contract.json',{});self.assertFalse(first_epoch_release(p,0));save(p/'jobs/0/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(p/'contract.json')));self.assertTrue(first_epoch_release(p,0));self.assertFalse(first_epoch_release(p,1));save(p/'global-stop.json',{})
|
||||
with self.assertRaises(RuntimeError):first_epoch_release(p,0)
|
||||
def test_full_reserves_then_settled_cost_once(self):
|
||||
self.assertEqual(project_total(self.c,{}, {})['projected_gpu_seconds'],118800)
|
||||
d=project_total(self.c,{}, {0:dict(gpu_seconds_charged=20000)})
|
||||
self.assertEqual(d['projected_gpu_seconds'],116480);self.assertEqual(d['projected_phase3_gpu_seconds'],189480)
|
||||
def test_validation_and_setup_in_projection(self):
|
||||
o=dict(epoch=1,charged=100,max_epoch_seconds=80,max_validation_seconds=30)
|
||||
d=project_cell(self.jobs[0],o);self.assertEqual(d['projected_gpu_seconds'],100+1.25*(199*80+20*30)+120)
|
||||
def test_boundary_overrun_stops(self):
|
||||
d=project_cell(self.jobs[0],dict(epoch=1,charged=100,max_epoch_seconds=100));self.assertFalse(d['within_cap'])
|
||||
self.assertEqual(project_total(self.c,{0:d},{})['status'],'STOP_BUDGET_PROJECTION')
|
||||
def test_parallel_dispatch_respects_held_gpus_and_identity(self):
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs,[],set(GPUS[:2]))],[0,1])
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[dict(self.jobs[0],gpu_uuid=GPUS[0]),dict(self.jobs[1],gpu_uuid=GPUS[1])],set(GPUS))],[2,3])
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[],{GPUS[0],GPUS[3]})],[2,3])
|
||||
def test_gpu2_waits_for_prior_track_postfit(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);[(r/n).mkdir() for n in ['item','train','post']];c=dict(item1_recovery_root=str(r/'item'),track_a_training_root=str(r/'train'),track_a_postfit_root=str(r/'post'))
|
||||
with patch('psutil.process_iter',return_value=[]):
|
||||
self.assertEqual(available_gpus(c),{GPUS[0]});save(r/'item/complete.json',{});save(r/'train/complete.json',{})
|
||||
self.assertEqual(available_gpus(c),{GPUS[0],GPUS[1],GPUS[3]});save(r/'post/complete.json',{});self.assertEqual(available_gpus(c),set(GPUS))
|
||||
def test_assignment_is_hash_bound_and_fold_specific(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);(r/'jobs/2').mkdir(parents=True);save(r/'contract.json',{});p=r/'jobs/2/assignment.json';save(p,dict(id=2,gpu_uuid=GPUS[3],contract_sha256=sha(r/'contract.json')))
|
||||
h=sha(p);self.assertEqual(assigned_job(r,self.c,2,h)['gpu_uuid'],GPUS[3]);save(p,dict(id=2,gpu_uuid=GPUS[0],contract_sha256=sha(r/'contract.json')))
|
||||
with self.assertRaises(ValueError):assigned_job(r,self.c,2,h)
|
||||
def guard_exercise(self,peak=100,monitor=0):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
q=Path(d);save(q/'contract.json',{});r=q/'jobs/0';r.mkdir(parents=True)
|
||||
j=dict(id=0,gpu_uuid='synthetic-seed1',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,target_steps=400,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)
|
||||
def work(*args,**kwargs):
|
||||
(r/'worker').mkdir();save(r/'worker/result.json',dict(status='COMPLETE',fold=0,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=monitor)
|
||||
with patch.object(guard,'config',return_value=(dict(jobs=[j]),{})),patch.object(guard,'assigned_job',return_value=j),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',work),patch.object(guard.time,'sleep'):
|
||||
rc=guard.run(q,0,'synthetic')
|
||||
return rc,json.loads((r/'complete.json').read_text())
|
||||
def test_guard_clean_complete(self):self.assertEqual(self.guard_exercise()[0],0)
|
||||
def test_guard_rejects_monitor_errors(self):self.assertIn('WORKER_MONITOR_ERRORS',self.guard_exercise(monitor=1)[1]['reasons'])
|
||||
def test_guard_preserves_excess_recorded_peak(self):self.assertIn('RECORDED_VRAM_LIMIT',self.guard_exercise(peak=16*2**30)[1]['reasons'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user