Publish item b shared-mode budget and wait-accounting delta for review

This commit is contained in:
NewMeltAssist committed 2026-11-19 01:33:52 +08:00
1 parent 482c490afb
commit 5bfef6a61a
16 files changed
+16580

No files matched your search

+62
View File
@@ -0,0 +1,62 @@
"""Shared-mode extrapolation with queue waiting charged separately from training."""
import math
MARGIN = 1.25
SETUP = 60
CLEANUP = 120
USER_CAP = 180000
RULING = '1795022093.308479'
def full_fold_bound(epoch_seconds, validation_seconds):
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
raise ValueError('Positive measured epoch time required')
if not math.isfinite(validation_seconds) or validation_seconds < 0:
raise ValueError('Invalid validation time')
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
def project(contract, charges, waits, histories, terminal):
folds = set(range(5))
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
raise ValueError('Cost ledger fold identity')
rows = {}
for i, charged in charges.items():
waiting = waits[i]
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
raise ValueError('Invalid queue and waiting charges')
elapsed = charged - waiting
job = contract['jobs'][i]
history = histories.get(i, [])
if i in terminal:
training = elapsed
elif history:
epochs = sorted(r['epoch'] for r in history)
finished = epochs[-1]
if epochs != list(range(1, finished + 1)) or finished > 200:
raise ValueError('Incomplete epoch history')
observed = [r['train_epoch_seconds'] for r in history]
diagnostics = [r.get('validation_seconds', 0) for r in history]
if any(not math.isfinite(x) or x <= 0 for x in observed):
raise ValueError('Invalid measured epoch')
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
raise ValueError('Invalid validation time')
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
validation_time = max(job['validation_seconds_bound'], *diagnostics)
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
else:
training = max(elapsed, job['full_training_gpu_seconds_bound'])
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
actual_training_with_setup_gpu_seconds=elapsed,
projected_training_gpu_seconds=training, projected=waiting+training)
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
+ sum(row['projected'] for row in rows.values()))
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)
+67
View File
@@ -0,0 +1,67 @@
"""b registration wrapper; unchanged reviewed shared admission and leases.
Only b's registered cap/ledger changes; the existing admission and reservations remain in use.
The live c module is imported without mutation, never patched or rewritten.
"""
import os,time,uuid
from pathlib import Path
import psutil
from phase3_shared_capacity import (registry,process_identity,alive,descendants,
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
def register_entry(root, digest, contract):
with registry() as d:
if 'b' in d['entries']:
raise FileExistsError('Sole item b entry already registered; no resume')
reservations = contract['phase_reservations']
cap = contract['gpu_cap']
if reservations['b'] != cap or sum(reservations.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=cap, phase_reservations=reservations, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['b']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
if l['state']=='RESERVED':
# owner is a queue with other children, not this job's guard.
# One independent slot and full VRAM/RSS remain reserved.
leases.append(dict(l,observed_pids=[],observed_rss=0))
continue
if l['state']!='GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
queue_owner=process_identity(os.getpid())
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)
+95
View File
@@ -0,0 +1,95 @@
"""Item b immutable scope, assignments and conservative total-cost projection."""
import json
import math
import statistics
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_seed1_v2_common import fold_view
from phase3_item_b_shared_budget import RULING, USER_CAP, full_fold_bound, project
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19')
def read(p):
return json.loads(Path(p).read_text())
def config(root):
if root.resolve() != ROOT:
raise PermissionError('Sole shared-mode b root')
c = read(root/'contract.json')
verify(root, 'implementation')
expected = dict(approval='1795011607.907609', seed=0, epochs=200, new_fits=5, shared_budget_ruling=RULING,
postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
phase3_gpu_cap=720000, no_retry=True)
if any(c.get(k) != v for k,v in expected.items()):
raise PermissionError('Fixed item_b scope/budgets')
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p,h)
bounds = read(c['shared_budget_evidence'])
if bounds['ruling'] != RULING or c['phase_reservations'] != bounds['phase_reservations']:
raise ValueError('Fixed shared budget identity')
full_bounds = []
for job, row in zip(c['jobs'], bounds['folds'], strict=True):
epoch = row['max_train_epoch_seconds']
full = full_fold_bound(epoch, job['validation_seconds_bound'])
if (job['id'] != row['fold'] or job['shared_epoch_seconds_bound'] != epoch
or job['full_training_gpu_seconds_bound'] != full):
raise ValueError('Fixed fold timing bounds')
full_bounds.append(full)
cap = math.ceil(c['preparation_charge_gpu_seconds'] + c['postfit_gpu_reserve']
+ sum(full_bounds) + max(full_bounds))
if (c['gpu_cap'] != cap or c['phase_reservations']['b'] != cap or cap > USER_CAP
or sum(c['phase_reservations'].values()) > c['phase3_gpu_cap']
or c['phase_reservations']['item1b'] != bounds['parked_1b_actual_including_tail']):
raise ValueError('User or phase 3 reservation exceeded')
pilot = read(c['pilot_contract'])
import numpy as np
import torch
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
raise ValueError('Original environment versions')
if [j['id'] for j in c['jobs']] != list(range(5)):
raise ValueError('Exactly five folds')
tests = []
for job in c['jobs']:
f = fold_view(pilot, job['id'])['folds'][0]
tests.extend(f['test'])
if job['target_steps'] != 200*len(f['learning']):
raise ValueError('Full fixed epochs')
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
if job[k] != f[k]:
raise ValueError('Original fold identity changed')
if len(tests) != 552 or len(set(tests)) != 552:
raise ValueError('Original552 roster')
rows=read(c['query_roster_contract'])['rows']
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
raise ValueError('552 parts /204 components query roster')
for r in rows:
p=pilot['parts'][r['case']]
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
raise ValueError('Learning query/geometry/cache identity')
if c['inputs']!=22 or c['added_parameters']!=384 or c['budget_approval']!='1795007580.895539' or not c['decision_requires_seed_reference']:
raise PermissionError('Fixed regional context scope')
checked(c['comparison_contract'], c['comparison_contract_sha256'])
return c,pilot
def assignment(root,c,index,digest):
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
raise PermissionError('Assignment identity')
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
def first_epoch_release(root,index):
if (root/'global-stop.json').exists():
raise RuntimeError('Global stop')
p = root/'jobs'/str(index)/'first-epoch-projection.json'
if not p.exists():
return False
d = read(p)
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
raise RuntimeError('First-epoch gate not passed')
return True
+136
View File
@@ -0,0 +1,136 @@
"""One item b attempt, scoped process memory plus failed-worker peak checks."""
import ctypes
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import threading
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase3_item_b_shared_common import config,assignment,read
from phase3_shared_capacity import validate_lease,alive
from track_a_recovery_monitor import Monitor
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
from s2_main_memory_guard import PeakRecords
def run(queue,index,digest):
parent=os.getppid()
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
raise OSError('PDEATHSIG')
requested=[]
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
if os.getppid()!=parent:
raise RuntimeError('Queue lost before setup')
c,_=config(queue);job=assignment(queue,c,index,digest)
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
remaining=c['runaway_gpu_seconds']-60
if remaining<90:
raise RuntimeError('No cleanup window before worker')
deadline=start+remaining
# Queue writes ready only after atomically transferring reservation ownership.
while not (root/'lease-ready.json').exists():
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
raise RuntimeError('Lease transfer timeout')
time.sleep(.1)
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
raise RuntimeError('Pinned queue lost before guard import completed')
os.sched_setaffinity(0,[26,27])
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
reasons=[];receipt=None
command=[sys.executable,'tools/phase3_item_b_shared_worker.py',str(queue),str(index),digest]
def reason(s):
if s not in reasons:reasons.append(s)
def workers():
result=[]
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
def kill():
for pid in workers():stop_group(pid)
def scan():
tracker.scan(root/'worker')
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
def local_watch():
while not done.is_set():
monitor.sample('recorded_peaks',scan)
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
if ok and not queue_live:reason('QUEUE_LOST')
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(.5)
def gpu_watch():
def sample():
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
rows=[]
for line in text.strip().splitlines():
if not line:continue
u,p,m=[x.strip() for x in line.split(',')]
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
own=set(workers())
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
while not done.is_set():
monitor.sample('gpu_processes',sample)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(1)
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
try:
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
for t in threads:t.start()
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
except BaseException as exc:
reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
for t in threads:
if t.ident is not None:t.join(timeout=12)
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
try:monitor.retry('final_peaks',scan,.1)
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
p=root/'worker/result.json';result=read(p) if p.exists() else None
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
if result:
from phase2_h1_worker_v2 import checked
checked(result['checkpoint'],result['checkpoint_sha256'])
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
elapsed=time.monotonic()-start
child=resource.getrusage(resource.RUSAGE_CHILDREN)
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
if 60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_PINS: '+str(exc))
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
+152
View File
@@ -0,0 +1,152 @@
"""Review-gated five-fold b training. Stops at frozen models, never postfit."""
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from phase3_item_b_shared_common import ROOT,config,read,project
from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
from track_a_recovery_monitor import Monitor
def run(root):
c,_=config(root);validate_review(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume')
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Explicit allocator required')
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
register_entry(root,sha(root/'contract.json'),c)
start=time.monotonic();identity=process_identity(os.getpid())
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
(root/'jobs').mkdir(exist_ok=False)
active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None
monitor=Monitor(root/'admission-errors.jsonl')
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
def charge(i):
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
def waiting_charges(charges):
return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()}
def projection_at_current_time():
charges = {i: charge(i) for i in range(5)}
return project(c, charges, waiting_charges(charges), histories, terminal)
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
try:
while active or pending:
if stops:raise RuntimeError(stops[0])
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
# Collect exits before granting any further admission.
for i,(p,lease,log,owner) in list(active.items()):
if p.poll() is None:continue
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
worker_identity=jr/'worker/worker-identity.json'
if worker_identity.exists() and alive(read(worker_identity)):
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None)
release(lease['token'],owner);del active[i]
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
for i in active:
folder=root/'jobs'/str(i)/'worker/history'
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
projection=projection_at_current_time()
save(root/'latest-cost-projection.json',projection)
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
for i in active:
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
if histories.get(i) and not gate.exists():
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending,
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
if pending:
i=pending[0]
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
if not ok:
time.sleep(1)
continue
lease,observation=decision
save(root/'latest-admission.json',observation)
# Acquisition includes bounded GPU queries; recheck cost before Popen.
fresh=projection_at_current_time()
if fresh['status']!='PASS':
if lease:release(lease['token'])
raise RuntimeError('CAP_DURING_ADMISSION')
if lease:
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
waits[i]=time.monotonic()-start
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
queue_identity=identity,pre_dispatch_wait_seconds=waits[i]))
log=(jr/'guard.log').open('x')
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')],
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
owner=process_identity(p.pid)
active[i]=(p,lease,log,owner);pending.remove(i)
transfer(lease['token'],owner)
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
time.sleep(1)
except BaseException as exc:
failure=type(exc).__name__+': '+str(exc);stop(failure)
finally:
if active:
stop(failure or 'QUEUE_CLEANUP')
for p,lease,log,owner in active.values():
if p.poll() is None:p.terminate()
deadline=time.monotonic()+45
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
for i,(p,lease,log,owner) in active.items():
if p.poll() is None:p.kill()
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],
guard_sha256=sha(cp) if cp.exists() else None)
wp=jr/'worker/worker-identity.json'
try:
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
release(lease['token'],owner)
except Exception as exc:
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
final_charges={i:charge(i) for i in range(5)}
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
files=[root/'contract.json']
if complete:
models=[]
for i in range(5):
r=read(root/'jobs'/str(i)/'worker/result.json')
models.append(dict(r));files.append(Path(r['checkpoint']))
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
save(root/'models.json',models);files.append(root/'models.json')
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE',
failure=failure,entries=list(terminal.values()),never_attempted=pending,
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()),
postfit_gpu_reserve=c['postfit_gpu_reserve'],
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
files.append(root/'complete.json')
freeze_start=time.monotonic()
freeze(root,'training-terminal' if complete else 'training-stop',files)
final_cost+=time.monotonic()-freeze_start
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
return 0 if complete and within else 1
if __name__=='__main__':raise SystemExit(run(ROOT))
+79
View File
@@ -0,0 +1,79 @@
"""One new S2 fold, fixed original200epoch schedule with seed0; no test prediction or target reads."""
import json,os,sys,time,traceback,random
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from phase2_h2_model import lr_epoch
from phase3_item_b_shared_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
from phase3_shared_capacity import process_identity
from phase3_item_b_data import standardization,load_part
from phase2_h2_data import to_device,arguments,cached
from phase3_item_b_model import H2Context as H2Geo
from phase2_h2_model import optimizer
from phase2_h2_pilot_worker import peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root,fold_id,assignment_sha):
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
save(out/'worker-identity.json',process_identity(os.getpid()))
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,c,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,c,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,fold_id):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':
import ctypes,signal
from phase3_shared_capacity import alive
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
parent=os.getppid()
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
+159
View File
@@ -0,0 +1,159 @@
"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
import ast
import fcntl
import math
import time
from pathlib import Path
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item_b_common import ROOT as ORIGINAL
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
from phase3_item_b_shared_common import ROOT, config, project, read
from track_a_h2_prepare import local_code_closure
BASE = Path(__file__).resolve().parents[1]
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
def measured_bounds(old_jobs):
c = read(C_ROOT / 'contract.json')
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
raise ValueError('c training identity changed')
for p, h in c['code_sha256'].items():
checked(p, h)
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
rows = []
for i, old in enumerate(old_jobs):
assignment = assignments[i]
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
raise ValueError('c fold assignment identity')
sample = samples[i]
seen = {r['pid'] for r in sample['rows']}
worker = workers[i]
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
raise ValueError('c worker missing from GPU sample')
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
and workers[j]['pid'] in seen]
if not peers:
raise ValueError('A live same-card c peer must corroborate the overlap')
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
candidates = []
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
mtime = p.stat().st_mtime
if mtime > sample['unix']:
continue
row = read(p)
seconds = row['train_epoch_seconds']
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
raise ValueError('c and b fold workload or epoch identity differs')
if mtime - seconds > overlap_start + 2:
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
sha256=sha(p), mtime=mtime))
if not candidates:
raise ValueError('No c epoch lies fully inside the same-card window')
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
max_train_epoch_seconds=maximum['epoch_seconds'],
validation_seconds_bound=old['validation_seconds_bound']))
return rows
def main():
start = time.process_time()
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
raise FileExistsError('Only a never-launched b may be rebound')
validate_review(ORIGINAL)
old = read(ORIGINAL / 'contract.json')
stop = read(PARKED / 'complete.json')
verify(PARKED, 'training-stop')
tail = read(PARKED / 'post-terminal-cost.json')
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
or any(e['status'] == 'PASS' for e in stop['entries'])
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
raise ValueError('1b terminal identity changed')
rows = measured_bounds(old['jobs'])
jobs = []
for job, row in zip(old['jobs'], rows, strict=True):
epoch = row['max_train_epoch_seconds']
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
wait_reserve = max(bounds)
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
original_worker = BASE / 'tools/phase3_item_b_worker.py'
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
'from phase3_item_b_common import')
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
raise ValueError('b worker differs beyond the run identity import')
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
BASE / 'tools/test_phase3_item_b_shared_budget.py']
codes = {**old['code_sha256'], **local_code_closure(files)}
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
'implementation-git-freeze.json', 'review.json')]
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
evidence = RECORDS / 'shared-cost-evidence.json'
if evidence.exists():
raise FileExistsError('A shared calibration receipt already exists')
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
aggregate_wait_reserve_gpu_seconds=wait_reserve,
wait_reserve_basis='one longest full-fold bound across all waiting charges',
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
worker_same_except_rebind_import=True))
sources = dict(old['source_sha256'])
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
jobs=jobs, code_sha256=codes, source_sha256=sources)
ROOT.mkdir()
save(ROOT / 'contract.json', contract)
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
if initial['status'] != 'PASS':
raise ValueError('Initial shared-mode projection cannot fit')
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
comparison_unchanged=True, scientific_worker_unchanged=True,
started=False, test_targets_read=False, review_adopted=False))
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
frozen = verify(ROOT, 'implementation')
config(ROOT)
save(ROOT / 'expected-review-binding.json', dict(
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
if __name__ == '__main__':
main()
+59
View File
@@ -0,0 +1,59 @@
"""CPU-only checks for the b wrapper's wait and measured-cost delta."""
from unittest import TestCase, main
from phase3_item_b_shared_budget import full_fold_bound, project
class SharedBudgetTests(TestCase):
def setUp(self):
self.contract = dict(
jobs=[dict(shared_epoch_seconds_bound=100, validation_seconds_bound=10,
full_training_gpu_seconds_bound=full_fold_bound(100, 10)) for _ in range(5)],
gpu_cap=170000, preparation_charge_gpu_seconds=600, postfit_gpu_reserve=7200,
runaway_gpu_seconds=60000,
)
def test_wait_changes_total_but_not_training_projection(self):
base = dict.fromkeys(range(5), 60)
waits = dict.fromkeys(range(5), 0)
measured = {0: [dict(epoch=1, train_epoch_seconds=103)]}
baseline = project(self.contract, base | {0: 185}, waits, measured, {})
result = project(self.contract, base | {0: 17185}, waits | {0: 17000}, measured, {})
self.assertEqual(baseline['folds']['0']['projected_training_gpu_seconds'],
result['folds']['0']['projected_training_gpu_seconds'])
self.assertEqual(result['projected_gpu_seconds']-baseline['projected_gpu_seconds'], 17000)
self.assertEqual(result['actual_waiting_gpu_seconds'], 17000)
def test_pending_wait_is_charged_once(self):
cost = dict.fromkeys(range(5), 3060)
wait = dict.fromkeys(range(5), 3000)
result = project(self.contract, cost, wait, {}, {})
bound = full_fold_bound(100, 10)
self.assertEqual(result['folds']['2']['projected_training_gpu_seconds'], bound)
self.assertEqual(result['projected_gpu_seconds'], 7800 + 5*(bound+3000))
def test_terminal_uses_actual_charge(self):
result = project(self.contract, dict.fromkeys(range(5), 60) | {4: 5000},
dict.fromkeys(range(5), 0) | {4: 2500}, {}, {4: {}})
self.assertEqual(result['folds']['4']['projected_training_gpu_seconds'], 2500)
self.assertEqual(result['folds']['4']['projected'], 5000)
def test_long_wait_does_not_hit_actual_training_runaway_limit(self):
costs = dict.fromkeys(range(5), 60)
waits = dict.fromkeys(range(5), 0)
contract = dict(self.contract, gpu_cap=1000000)
self.assertEqual(project(contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'PASS')
self.assertEqual(project(contract, costs | {0: 80000}, waits, {}, {})['status'], 'STOP_BUDGET')
self.assertEqual(project(self.contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'STOP_BUDGET')
def test_observed_b_epoch_can_raise_shared_bound(self):
costs = dict.fromkeys(range(5), 60) | {0: 220}
waits = dict.fromkeys(range(5), 0)
faster = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=90)]}, {})
slower = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=130)]}, {})
self.assertEqual(slower['projected_gpu_seconds']-faster['projected_gpu_seconds'], 1.25*199*30)
if __name__ == '__main__':
main()