Publish item b shared-mode budget and wait-accounting delta for review
This commit is contained in:
1 parent
482c490afb
commit
5bfef6a61a
16 files changed
+16580
No files matched your search
@@ -0,0 +1,62 @@
|
||||
"""Shared-mode extrapolation with queue waiting charged separately from training."""
|
||||
|
||||
import math
|
||||
|
||||
|
||||
MARGIN = 1.25
|
||||
SETUP = 60
|
||||
CLEANUP = 120
|
||||
USER_CAP = 180000
|
||||
RULING = '1795022093.308479'
|
||||
|
||||
|
||||
def full_fold_bound(epoch_seconds, validation_seconds):
|
||||
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
|
||||
raise ValueError('Positive measured epoch time required')
|
||||
if not math.isfinite(validation_seconds) or validation_seconds < 0:
|
||||
raise ValueError('Invalid validation time')
|
||||
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
|
||||
|
||||
|
||||
def project(contract, charges, waits, histories, terminal):
|
||||
folds = set(range(5))
|
||||
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
|
||||
raise ValueError('Cost ledger fold identity')
|
||||
rows = {}
|
||||
for i, charged in charges.items():
|
||||
waiting = waits[i]
|
||||
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
|
||||
raise ValueError('Invalid queue and waiting charges')
|
||||
elapsed = charged - waiting
|
||||
job = contract['jobs'][i]
|
||||
history = histories.get(i, [])
|
||||
if i in terminal:
|
||||
training = elapsed
|
||||
elif history:
|
||||
epochs = sorted(r['epoch'] for r in history)
|
||||
finished = epochs[-1]
|
||||
if epochs != list(range(1, finished + 1)) or finished > 200:
|
||||
raise ValueError('Incomplete epoch history')
|
||||
observed = [r['train_epoch_seconds'] for r in history]
|
||||
diagnostics = [r.get('validation_seconds', 0) for r in history]
|
||||
if any(not math.isfinite(x) or x <= 0 for x in observed):
|
||||
raise ValueError('Invalid measured epoch')
|
||||
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
|
||||
raise ValueError('Invalid validation time')
|
||||
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
|
||||
validation_time = max(job['validation_seconds_bound'], *diagnostics)
|
||||
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
|
||||
else:
|
||||
training = max(elapsed, job['full_training_gpu_seconds_bound'])
|
||||
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
|
||||
actual_training_with_setup_gpu_seconds=elapsed,
|
||||
projected_training_gpu_seconds=training, projected=waiting+training)
|
||||
|
||||
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
|
||||
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
|
||||
+ sum(row['projected'] for row in rows.values()))
|
||||
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
|
||||
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
|
||||
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
|
||||
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
|
||||
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""b registration wrapper; unchanged reviewed shared admission and leases.
|
||||
|
||||
Only b's registered cap/ledger changes; the existing admission and reservations remain in use.
|
||||
The live c module is imported without mutation, never patched or rewritten.
|
||||
"""
|
||||
import os,time,uuid
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase3_shared_capacity import (registry,process_identity,alive,descendants,
|
||||
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
|
||||
|
||||
def register_entry(root, digest, contract):
|
||||
with registry() as d:
|
||||
if 'b' in d['entries']:
|
||||
raise FileExistsError('Sole item b entry already registered; no resume')
|
||||
reservations = contract['phase_reservations']
|
||||
cap = contract['gpu_cap']
|
||||
if reservations['b'] != cap or sum(reservations.values()) > 720000:
|
||||
raise RuntimeError('Whole phase3 reserved cap')
|
||||
d['entries']['b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
||||
cap=cap, phase_reservations=reservations, started_unix=time.time())
|
||||
|
||||
|
||||
def acquire(root, fold, digest):
|
||||
with registry() as d:
|
||||
entry = d['entries']['b']
|
||||
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
||||
raise PermissionError('Entry owner/identity')
|
||||
leases = []
|
||||
for l in d['leases'].values():
|
||||
# Normal guard exit can race this observation. Block admission until
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
if l['state']=='RESERVED':
|
||||
# owner is a queue with other children, not this job's guard.
|
||||
# One independent slot and full VRAM/RSS remain reserved.
|
||||
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
||||
continue
|
||||
if l['state']!='GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
rss=0
|
||||
for pid in ids:
|
||||
try:rss+=psutil.Process(pid).memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
||||
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
||||
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
||||
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
||||
if not cpus:
|
||||
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
||||
for gpu in GPUS[:3]:
|
||||
ok, reason = admission(s, leases, gpu)
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
queue_owner=process_identity(os.getpid())
|
||||
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
return None, dict(reason='NO_CAPACITY', snapshot=s)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Item b immutable scope, assignments and conservative total-cost projection."""
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import fold_view
|
||||
|
||||
from phase3_item_b_shared_budget import RULING, USER_CAP, full_fold_bound, project
|
||||
|
||||
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19')
|
||||
|
||||
|
||||
def read(p):
|
||||
return json.loads(Path(p).read_text())
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Sole shared-mode b root')
|
||||
c = read(root/'contract.json')
|
||||
verify(root, 'implementation')
|
||||
expected = dict(approval='1795011607.907609', seed=0, epochs=200, new_fits=5, shared_budget_ruling=RULING,
|
||||
postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
|
||||
phase3_gpu_cap=720000, no_retry=True)
|
||||
if any(c.get(k) != v for k,v in expected.items()):
|
||||
raise PermissionError('Fixed item_b scope/budgets')
|
||||
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p,h)
|
||||
bounds = read(c['shared_budget_evidence'])
|
||||
if bounds['ruling'] != RULING or c['phase_reservations'] != bounds['phase_reservations']:
|
||||
raise ValueError('Fixed shared budget identity')
|
||||
full_bounds = []
|
||||
for job, row in zip(c['jobs'], bounds['folds'], strict=True):
|
||||
epoch = row['max_train_epoch_seconds']
|
||||
full = full_fold_bound(epoch, job['validation_seconds_bound'])
|
||||
if (job['id'] != row['fold'] or job['shared_epoch_seconds_bound'] != epoch
|
||||
or job['full_training_gpu_seconds_bound'] != full):
|
||||
raise ValueError('Fixed fold timing bounds')
|
||||
full_bounds.append(full)
|
||||
cap = math.ceil(c['preparation_charge_gpu_seconds'] + c['postfit_gpu_reserve']
|
||||
+ sum(full_bounds) + max(full_bounds))
|
||||
if (c['gpu_cap'] != cap or c['phase_reservations']['b'] != cap or cap > USER_CAP
|
||||
or sum(c['phase_reservations'].values()) > c['phase3_gpu_cap']
|
||||
or c['phase_reservations']['item1b'] != bounds['parked_1b_actual_including_tail']):
|
||||
raise ValueError('User or phase 3 reservation exceeded')
|
||||
pilot = read(c['pilot_contract'])
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
|
||||
raise ValueError('Original environment versions')
|
||||
if [j['id'] for j in c['jobs']] != list(range(5)):
|
||||
raise ValueError('Exactly five folds')
|
||||
tests = []
|
||||
for job in c['jobs']:
|
||||
f = fold_view(pilot, job['id'])['folds'][0]
|
||||
tests.extend(f['test'])
|
||||
if job['target_steps'] != 200*len(f['learning']):
|
||||
raise ValueError('Full fixed epochs')
|
||||
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
|
||||
if job[k] != f[k]:
|
||||
raise ValueError('Original fold identity changed')
|
||||
if len(tests) != 552 or len(set(tests)) != 552:
|
||||
raise ValueError('Original552 roster')
|
||||
rows=read(c['query_roster_contract'])['rows']
|
||||
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
|
||||
raise ValueError('552 parts /204 components query roster')
|
||||
for r in rows:
|
||||
p=pilot['parts'][r['case']]
|
||||
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
|
||||
raise ValueError('Learning query/geometry/cache identity')
|
||||
if c['inputs']!=22 or c['added_parameters']!=384 or c['budget_approval']!='1795007580.895539' or not c['decision_requires_seed_reference']:
|
||||
raise PermissionError('Fixed regional context scope')
|
||||
checked(c['comparison_contract'], c['comparison_contract_sha256'])
|
||||
return c,pilot
|
||||
|
||||
|
||||
def assignment(root,c,index,digest):
|
||||
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
|
||||
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise PermissionError('Assignment identity')
|
||||
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
|
||||
|
||||
|
||||
def first_epoch_release(root,index):
|
||||
if (root/'global-stop.json').exists():
|
||||
raise RuntimeError('Global stop')
|
||||
p = root/'jobs'/str(index)/'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
d = read(p)
|
||||
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise RuntimeError('First-epoch gate not passed')
|
||||
return True
|
||||
@@ -0,0 +1,136 @@
|
||||
"""One item b attempt, scoped process memory plus failed-worker peak checks."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase3_item_b_shared_common import config,assignment,read
|
||||
from phase3_shared_capacity import validate_lease,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,index,digest):
|
||||
parent=os.getppid()
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||||
raise OSError('PDEATHSIG')
|
||||
requested=[]
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||||
if os.getppid()!=parent:
|
||||
raise RuntimeError('Queue lost before setup')
|
||||
c,_=config(queue);job=assignment(queue,c,index,digest)
|
||||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||||
remaining=c['runaway_gpu_seconds']-60
|
||||
if remaining<90:
|
||||
raise RuntimeError('No cleanup window before worker')
|
||||
deadline=start+remaining
|
||||
# Queue writes ready only after atomically transferring reservation ownership.
|
||||
while not (root/'lease-ready.json').exists():
|
||||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||||
raise RuntimeError('Lease transfer timeout')
|
||||
time.sleep(.1)
|
||||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||||
os.sched_setaffinity(0,[26,27])
|
||||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
reasons=[];receipt=None
|
||||
command=[sys.executable,'tools/phase3_item_b_shared_worker.py',str(queue),str(index),digest]
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def workers():
|
||||
result=[]
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return result
|
||||
def kill():
|
||||
for pid in workers():stop_group(pid)
|
||||
def scan():
|
||||
tracker.scan(root/'worker')
|
||||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||||
if ok and not queue_live:reason('QUEUE_LOST')
|
||||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
def gpu_watch():
|
||||
def sample():
|
||||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||||
rows=[]
|
||||
for line in text.strip().splitlines():
|
||||
if not line:continue
|
||||
u,p,m=[x.strip() for x in line.split(',')]
|
||||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||||
own=set(workers())
|
||||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||||
while not done.is_set():
|
||||
monitor.sample('gpu_processes',sample)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(1)
|
||||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||||
try:
|
||||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||||
for t in threads:t.start()
|
||||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:
|
||||
reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:
|
||||
if t.ident is not None:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('final_peaks',scan,.1)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
p=root/'worker/result.json';result=read(p) if p.exists() else None
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
|
||||
if result:
|
||||
from phase2_h1_worker_v2 import checked
|
||||
checked(result['checkpoint'],result['checkpoint_sha256'])
|
||||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||||
if 60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
|
||||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Review-gated five-fold b training. Stops at frozen models, never postfit."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_item_b_shared_common import ROOT,config,read,project
|
||||
from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);validate_review(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume')
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Explicit allocator required')
|
||||
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
|
||||
register_entry(root,sha(root/'contract.json'),c)
|
||||
start=time.monotonic();identity=process_identity(os.getpid())
|
||||
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None
|
||||
monitor=Monitor(root/'admission-errors.jsonl')
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
|
||||
def charge(i):
|
||||
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
|
||||
def waiting_charges(charges):
|
||||
return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()}
|
||||
|
||||
def projection_at_current_time():
|
||||
charges = {i: charge(i) for i in range(5)}
|
||||
return project(c, charges, waiting_charges(charges), histories, terminal)
|
||||
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
try:
|
||||
while active or pending:
|
||||
if stops:raise RuntimeError(stops[0])
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
|
||||
# Collect exits before granting any further admission.
|
||||
for i,(p,lease,log,owner) in list(active.items()):
|
||||
if p.poll() is None:continue
|
||||
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
worker_identity=jr/'worker/worker-identity.json'
|
||||
if worker_identity.exists() and alive(read(worker_identity)):
|
||||
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
|
||||
gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None)
|
||||
release(lease['token'],owner);del active[i]
|
||||
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
|
||||
for i in active:
|
||||
folder=root/'jobs'/str(i)/'worker/history'
|
||||
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
|
||||
projection=projection_at_current_time()
|
||||
save(root/'latest-cost-projection.json',projection)
|
||||
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
|
||||
for i in active:
|
||||
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
|
||||
if histories.get(i) and not gate.exists():
|
||||
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
|
||||
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending,
|
||||
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
|
||||
if pending:
|
||||
i=pending[0]
|
||||
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
|
||||
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
|
||||
if not ok:
|
||||
time.sleep(1)
|
||||
continue
|
||||
lease,observation=decision
|
||||
save(root/'latest-admission.json',observation)
|
||||
# Acquisition includes bounded GPU queries; recheck cost before Popen.
|
||||
fresh=projection_at_current_time()
|
||||
if fresh['status']!='PASS':
|
||||
if lease:release(lease['token'])
|
||||
raise RuntimeError('CAP_DURING_ADMISSION')
|
||||
if lease:
|
||||
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
|
||||
waits[i]=time.monotonic()-start
|
||||
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
|
||||
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
|
||||
queue_identity=identity,pre_dispatch_wait_seconds=waits[i]))
|
||||
log=(jr/'guard.log').open('x')
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
||||
p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')],
|
||||
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
owner=process_identity(p.pid)
|
||||
active[i]=(p,lease,log,owner);pending.remove(i)
|
||||
transfer(lease['token'],owner)
|
||||
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
|
||||
time.sleep(1)
|
||||
except BaseException as exc:
|
||||
failure=type(exc).__name__+': '+str(exc);stop(failure)
|
||||
finally:
|
||||
if active:
|
||||
stop(failure or 'QUEUE_CLEANUP')
|
||||
for p,lease,log,owner in active.values():
|
||||
if p.poll() is None:p.terminate()
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
|
||||
for i,(p,lease,log,owner) in active.items():
|
||||
if p.poll() is None:p.kill()
|
||||
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],
|
||||
guard_sha256=sha(cp) if cp.exists() else None)
|
||||
wp=jr/'worker/worker-identity.json'
|
||||
try:
|
||||
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
|
||||
release(lease['token'],owner)
|
||||
except Exception as exc:
|
||||
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
|
||||
final_charges={i:charge(i) for i in range(5)}
|
||||
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
|
||||
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
|
||||
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
|
||||
files=[root/'contract.json']
|
||||
if complete:
|
||||
models=[]
|
||||
for i in range(5):
|
||||
r=read(root/'jobs'/str(i)/'worker/result.json')
|
||||
models.append(dict(r));files.append(Path(r['checkpoint']))
|
||||
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
|
||||
save(root/'models.json',models);files.append(root/'models.json')
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE',
|
||||
failure=failure,entries=list(terminal.values()),never_attempted=pending,
|
||||
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
|
||||
cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()),
|
||||
postfit_gpu_reserve=c['postfit_gpu_reserve'],
|
||||
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
|
||||
files.append(root/'complete.json')
|
||||
freeze_start=time.monotonic()
|
||||
freeze(root,'training-terminal' if complete else 'training-stop',files)
|
||||
final_cost+=time.monotonic()-freeze_start
|
||||
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
|
||||
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
|
||||
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
|
||||
return 0 if complete and within else 1
|
||||
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(ROOT))
|
||||
@@ -0,0 +1,79 @@
|
||||
"""One new S2 fold, fixed original200epoch schedule with seed0; no test prediction or target reads."""
|
||||
import json,os,sys,time,traceback,random
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from phase2_h2_model import lr_epoch
|
||||
from phase3_item_b_shared_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
|
||||
from phase3_shared_capacity import process_identity
|
||||
from phase3_item_b_data import standardization,load_part
|
||||
from phase2_h2_data import to_device,arguments,cached
|
||||
from phase3_item_b_model import H2Context as H2Geo
|
||||
from phase2_h2_model import optimizer
|
||||
from phase2_h2_pilot_worker import peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id,assignment_sha):
|
||||
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
save(out/'worker-identity.json',process_identity(os.getpid()))
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,c,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,c,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':
|
||||
import ctypes,signal
|
||||
from phase3_shared_capacity import alive
|
||||
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
|
||||
parent=os.getppid()
|
||||
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
|
||||
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
|
||||
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
|
||||
|
||||
import ast
|
||||
import fcntl
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_item_b_common import ROOT as ORIGINAL
|
||||
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
|
||||
from phase3_item_b_shared_common import ROOT, config, project, read
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
|
||||
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
|
||||
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
|
||||
|
||||
|
||||
def measured_bounds(old_jobs):
|
||||
c = read(C_ROOT / 'contract.json')
|
||||
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
|
||||
raise ValueError('c training identity changed')
|
||||
for p, h in c['code_sha256'].items():
|
||||
checked(p, h)
|
||||
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
|
||||
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
|
||||
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
|
||||
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
|
||||
rows = []
|
||||
for i, old in enumerate(old_jobs):
|
||||
assignment = assignments[i]
|
||||
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
|
||||
raise ValueError('c fold assignment identity')
|
||||
sample = samples[i]
|
||||
seen = {r['pid'] for r in sample['rows']}
|
||||
worker = workers[i]
|
||||
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
|
||||
raise ValueError('c worker missing from GPU sample')
|
||||
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
|
||||
and workers[j]['pid'] in seen]
|
||||
if not peers:
|
||||
raise ValueError('A live same-card c peer must corroborate the overlap')
|
||||
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
|
||||
candidates = []
|
||||
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
|
||||
mtime = p.stat().st_mtime
|
||||
if mtime > sample['unix']:
|
||||
continue
|
||||
row = read(p)
|
||||
seconds = row['train_epoch_seconds']
|
||||
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
|
||||
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
|
||||
raise ValueError('c and b fold workload or epoch identity differs')
|
||||
if mtime - seconds > overlap_start + 2:
|
||||
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
|
||||
sha256=sha(p), mtime=mtime))
|
||||
if not candidates:
|
||||
raise ValueError('No c epoch lies fully inside the same-card window')
|
||||
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
|
||||
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
|
||||
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
|
||||
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
|
||||
max_train_epoch_seconds=maximum['epoch_seconds'],
|
||||
validation_seconds_bound=old['validation_seconds_bound']))
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
start = time.process_time()
|
||||
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
|
||||
raise FileExistsError('Only a never-launched b may be rebound')
|
||||
validate_review(ORIGINAL)
|
||||
old = read(ORIGINAL / 'contract.json')
|
||||
stop = read(PARKED / 'complete.json')
|
||||
verify(PARKED, 'training-stop')
|
||||
tail = read(PARKED / 'post-terminal-cost.json')
|
||||
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
|
||||
or any(e['status'] == 'PASS' for e in stop['entries'])
|
||||
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
|
||||
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
|
||||
raise ValueError('1b terminal identity changed')
|
||||
rows = measured_bounds(old['jobs'])
|
||||
jobs = []
|
||||
for job, row in zip(old['jobs'], rows, strict=True):
|
||||
epoch = row['max_train_epoch_seconds']
|
||||
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
|
||||
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
|
||||
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
|
||||
wait_reserve = max(bounds)
|
||||
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
|
||||
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
|
||||
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
|
||||
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
|
||||
original_worker = BASE / 'tools/phase3_item_b_worker.py'
|
||||
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
|
||||
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
|
||||
'from phase3_item_b_common import')
|
||||
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
|
||||
raise ValueError('b worker differs beyond the run identity import')
|
||||
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
|
||||
BASE / 'tools/test_phase3_item_b_shared_budget.py']
|
||||
codes = {**old['code_sha256'], **local_code_closure(files)}
|
||||
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
|
||||
'implementation-git-freeze.json', 'review.json')]
|
||||
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
|
||||
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
|
||||
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
evidence = RECORDS / 'shared-cost-evidence.json'
|
||||
if evidence.exists():
|
||||
raise FileExistsError('A shared calibration receipt already exists')
|
||||
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
|
||||
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
|
||||
aggregate_wait_reserve_gpu_seconds=wait_reserve,
|
||||
wait_reserve_basis='one longest full-fold bound across all waiting charges',
|
||||
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
|
||||
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
|
||||
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
|
||||
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
|
||||
worker_same_except_rebind_import=True))
|
||||
sources = dict(old['source_sha256'])
|
||||
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
|
||||
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
|
||||
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
|
||||
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
|
||||
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
|
||||
jobs=jobs, code_sha256=codes, source_sha256=sources)
|
||||
ROOT.mkdir()
|
||||
save(ROOT / 'contract.json', contract)
|
||||
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
|
||||
if initial['status'] != 'PASS':
|
||||
raise ValueError('Initial shared-mode projection cannot fit')
|
||||
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
|
||||
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
|
||||
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
|
||||
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
|
||||
comparison_unchanged=True, scientific_worker_unchanged=True,
|
||||
started=False, test_targets_read=False, review_adopted=False))
|
||||
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
|
||||
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
|
||||
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
|
||||
frozen = verify(ROOT, 'implementation')
|
||||
config(ROOT)
|
||||
save(ROOT / 'expected-review-binding.json', dict(
|
||||
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
|
||||
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
|
||||
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
|
||||
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
|
||||
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""CPU-only checks for the b wrapper's wait and measured-cost delta."""
|
||||
|
||||
from unittest import TestCase, main
|
||||
|
||||
from phase3_item_b_shared_budget import full_fold_bound, project
|
||||
|
||||
|
||||
class SharedBudgetTests(TestCase):
|
||||
def setUp(self):
|
||||
self.contract = dict(
|
||||
jobs=[dict(shared_epoch_seconds_bound=100, validation_seconds_bound=10,
|
||||
full_training_gpu_seconds_bound=full_fold_bound(100, 10)) for _ in range(5)],
|
||||
gpu_cap=170000, preparation_charge_gpu_seconds=600, postfit_gpu_reserve=7200,
|
||||
runaway_gpu_seconds=60000,
|
||||
)
|
||||
|
||||
def test_wait_changes_total_but_not_training_projection(self):
|
||||
base = dict.fromkeys(range(5), 60)
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
measured = {0: [dict(epoch=1, train_epoch_seconds=103)]}
|
||||
baseline = project(self.contract, base | {0: 185}, waits, measured, {})
|
||||
result = project(self.contract, base | {0: 17185}, waits | {0: 17000}, measured, {})
|
||||
self.assertEqual(baseline['folds']['0']['projected_training_gpu_seconds'],
|
||||
result['folds']['0']['projected_training_gpu_seconds'])
|
||||
self.assertEqual(result['projected_gpu_seconds']-baseline['projected_gpu_seconds'], 17000)
|
||||
self.assertEqual(result['actual_waiting_gpu_seconds'], 17000)
|
||||
|
||||
def test_pending_wait_is_charged_once(self):
|
||||
cost = dict.fromkeys(range(5), 3060)
|
||||
wait = dict.fromkeys(range(5), 3000)
|
||||
result = project(self.contract, cost, wait, {}, {})
|
||||
bound = full_fold_bound(100, 10)
|
||||
self.assertEqual(result['folds']['2']['projected_training_gpu_seconds'], bound)
|
||||
self.assertEqual(result['projected_gpu_seconds'], 7800 + 5*(bound+3000))
|
||||
|
||||
def test_terminal_uses_actual_charge(self):
|
||||
result = project(self.contract, dict.fromkeys(range(5), 60) | {4: 5000},
|
||||
dict.fromkeys(range(5), 0) | {4: 2500}, {}, {4: {}})
|
||||
self.assertEqual(result['folds']['4']['projected_training_gpu_seconds'], 2500)
|
||||
self.assertEqual(result['folds']['4']['projected'], 5000)
|
||||
|
||||
def test_long_wait_does_not_hit_actual_training_runaway_limit(self):
|
||||
costs = dict.fromkeys(range(5), 60)
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
contract = dict(self.contract, gpu_cap=1000000)
|
||||
self.assertEqual(project(contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'PASS')
|
||||
self.assertEqual(project(contract, costs | {0: 80000}, waits, {}, {})['status'], 'STOP_BUDGET')
|
||||
self.assertEqual(project(self.contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'STOP_BUDGET')
|
||||
|
||||
def test_observed_b_epoch_can_raise_shared_bound(self):
|
||||
costs = dict.fromkeys(range(5), 60) | {0: 220}
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
faster = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=90)]}, {})
|
||||
slower = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=130)]}, {})
|
||||
self.assertEqual(slower['projected_gpu_seconds']-faster['projected_gpu_seconds'], 1.25*199*30)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user