Freeze approved H2 S1 fold0 last-epoch200 training amendment
This commit is contained in:
1 parent
a864b29563
commit
7a7174746d
8 files changed
+345
No files matched your search
@@ -0,0 +1,30 @@
|
||||
"""Labelled S1 amendment; original pilot and scientific sources remain immutable."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked
|
||||
from phase2_h2_data import config as pilot_config
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
||||
old=pilot_config(Path(c['pilot_root']))
|
||||
if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed')
|
||||
if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap')
|
||||
return c,old
|
||||
|
||||
def gate(field_high,last_high):
|
||||
return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0))
|
||||
|
||||
def selected_checkpoint(completed_epochs):
|
||||
if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated')
|
||||
return 'last-epoch200.pt'
|
||||
|
||||
def stats_from_pilot(c,old):
|
||||
stats=read(checked(c['standardization_path'],c['standardization_sha256']))
|
||||
if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch')
|
||||
if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization')
|
||||
return stats
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Independent bounded S1 train guard; original pilot remains immutable."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h2_s1_common import config
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'s1.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
|
||||
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(root)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT')
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Start the one published H2 S1 training after host/GPU duplicate checks."""
|
||||
import hashlib,json,pathlib,subprocess,sys,time,psutil
|
||||
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17'
|
||||
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
|
||||
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
contract=json.loads((root/'contract.json').read_text())
|
||||
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
c=p.info['cmdline'] or []
|
||||
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2'
|
||||
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
|
||||
uuid=contract['gpu_uuid']
|
||||
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
|
||||
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
|
||||
unit='meltmind-phase2-h2-s1-20261117'
|
||||
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
|
||||
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)]
|
||||
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
|
||||
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
|
||||
print('SERVICE_LAUNCHED',unit,flush=True)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels."""
|
||||
import json,os,sys,time,traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import seed,peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root):
|
||||
c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stats=stats_from_pilot(c,old);save(out/'standardization.json',stats)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Engineering regressions for the independent S1 scope, no real data/GPU."""
|
||||
import ast,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import phase2_h2_s1_common as common
|
||||
from phase2_h2_model import lr_epoch
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
class S1Tests(unittest.TestCase):
|
||||
def test_gate_strict_and_or(self):
|
||||
self.assertFalse(common.gate(.005,0));self.assertTrue(common.gate(.00499,1));self.assertTrue(common.gate(.1,-1e-10));self.assertFalse(common.gate(None,float('nan')))
|
||||
def test_final_only(self):
|
||||
for e in [1,42,199,201]:
|
||||
with self.assertRaises(ValueError):common.selected_checkpoint(e)
|
||||
self.assertEqual(common.selected_checkpoint(200),'last-epoch200.pt')
|
||||
def test_normalization_rejects_validation(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d)/'s.json';p.write_text(json.dumps(dict(parts=['validation'],nodes=1,mean=[0]*19,scale=[1]*19)))
|
||||
c=dict(standardization_path=str(p),standardization_sha256='dummy')
|
||||
with patch.object(common,'checked',side_effect=lambda p,h:Path(p)):
|
||||
with self.assertRaises(ValueError):common.stats_from_pilot(c,dict(folds=[dict(learning=['learning'],learning_nodes=1)]))
|
||||
def test_schedule_index(self):
|
||||
self.assertAlmostEqual(lr_epoch(1),.0001);self.assertAlmostEqual(lr_epoch(10),.001);self.assertAlmostEqual(lr_epoch(200),.00001);self.assertLess(lr_epoch(11),lr_epoch(10))
|
||||
def test_history_peak_cannot_be_hidden(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);(p/'history').mkdir();(p/'history/001.json').write_text(json.dumps(dict(peak_reserved_bytes=16*2**30,peak_allocated_bytes=1)))
|
||||
(p/'result.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=1)))
|
||||
tracker=PeakRecords();self.assertTrue(tracker.scan(p));(p/'history/001.json').unlink();self.assertTrue(tracker.scan(p))
|
||||
def test_train_does_not_open_test_or_pilot_weights(self):
|
||||
tree=ast.parse(Path('tools/phase2_h2_s1_worker.py').read_text())
|
||||
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call)]
|
||||
self.assertFalse(any(isinstance(n.func,ast.Attribute) and n.func.attr=='load' and isinstance(n.func.value,ast.Name) and n.func.value.id=='torch' for n in calls))
|
||||
roles=[n.args[3].value for n in calls if isinstance(n.func,ast.Name) and n.func.id=='load_part']
|
||||
self.assertEqual(sorted(roles),['learning','validation'])
|
||||
def test_unapproved_scope_rejected_before_sources(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);(p/'contract.json').write_text(json.dumps(dict(approval='wrong',mode='H2_S1_FOLD0_LAST200')))
|
||||
with self.assertRaises(PermissionError):common.config(p)
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user