Freeze approved H2 S1 fold0 last-epoch200 training amendment

This commit is contained in:
Frank14f committed 2026-11-17 16:08:56 +08:00
1 parent a864b29563
commit 7a7174746d
8 files changed
+345

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
"""Labelled S1 amendment; original pilot and scientific sources remain immutable."""
import json
from pathlib import Path
import numpy as np
import torch
from phase2_h1_worker_v2 import checked
from phase2_h2_data import config as pilot_config
BASE=Path('/home/frank14f/optane/MeltMind')
def read(p):return json.loads(Path(p).read_text())
def config(root):
c=read(root/'contract.json')
if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
old=pilot_config(Path(c['pilot_root']))
if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed')
if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap')
return c,old
def gate(field_high,last_high):
return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0))
def selected_checkpoint(completed_epochs):
if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated')
return 'last-epoch200.pt'
def stats_from_pilot(c,old):
stats=read(checked(c['standardization_path'],c['standardization_sha256']))
if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch')
if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization')
return stats
+83
View File
@@ -0,0 +1,83 @@
"""Independent bounded S1 train guard; original pilot remains immutable."""
import fcntl
import json
import os
from pathlib import Path
import signal
import sys
import threading
import time
import psutil
from phase2_h2_s1_common import config
from phase2_h1_worker_v2 import save,sha
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def run(root):
c,_=config(root);os.sched_setaffinity(0,c['guard_cpus'])
with (root/'s1.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry')
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)]
def kill_worker():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
def scan():
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT')
for x in requested:reason(x)
save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
if reasons:kill_worker()
done.wait(.5)
thread=None
try:
for i in range(3):
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
if i<2:time.sleep(2)
save(root/'idle-polls.json',samples)
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=c['gpu_seconds']-elapsed()-10
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
for s,handler in old.items():signal.signal(s,handler)
value=None;p=root/'worker/result.json'
if p.exists():value=json.loads(p.read_text())
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(root)
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT')
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors,
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu,
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
+20
View File
@@ -0,0 +1,20 @@
"""Start the one published H2 S1 training after host/GPU duplicate checks."""
import hashlib,json,pathlib,subprocess,sys,time,psutil
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17'
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
contract=json.loads((root/'contract.json').read_text())
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
for p in psutil.process_iter(['pid','cmdline']):
c=p.info['cmdline'] or []
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2'
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
uuid=contract['gpu_uuid']
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
unit='meltmind-phase2-h2-s1-20261117'
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)]
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
print('SERVICE_LAUNCHED',unit,flush=True)
+62
View File
@@ -0,0 +1,62 @@
"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels."""
import json,os,sys,time,traceback
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import seed,peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root):
c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
def record(item,epoch_file=False):
item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stats=stats_from_pilot(c,old);save(out/'standardization.json',stats)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]))
+38
View File
@@ -0,0 +1,38 @@
"""Engineering regressions for the independent S1 scope, no real data/GPU."""
import ast,json,tempfile,unittest
from pathlib import Path
from unittest.mock import patch
import phase2_h2_s1_common as common
from phase2_h2_model import lr_epoch
from s2_main_memory_guard import PeakRecords
class S1Tests(unittest.TestCase):
def test_gate_strict_and_or(self):
self.assertFalse(common.gate(.005,0));self.assertTrue(common.gate(.00499,1));self.assertTrue(common.gate(.1,-1e-10));self.assertFalse(common.gate(None,float('nan')))
def test_final_only(self):
for e in [1,42,199,201]:
with self.assertRaises(ValueError):common.selected_checkpoint(e)
self.assertEqual(common.selected_checkpoint(200),'last-epoch200.pt')
def test_normalization_rejects_validation(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'s.json';p.write_text(json.dumps(dict(parts=['validation'],nodes=1,mean=[0]*19,scale=[1]*19)))
c=dict(standardization_path=str(p),standardization_sha256='dummy')
with patch.object(common,'checked',side_effect=lambda p,h:Path(p)):
with self.assertRaises(ValueError):common.stats_from_pilot(c,dict(folds=[dict(learning=['learning'],learning_nodes=1)]))
def test_schedule_index(self):
self.assertAlmostEqual(lr_epoch(1),.0001);self.assertAlmostEqual(lr_epoch(10),.001);self.assertAlmostEqual(lr_epoch(200),.00001);self.assertLess(lr_epoch(11),lr_epoch(10))
def test_history_peak_cannot_be_hidden(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);(p/'history').mkdir();(p/'history/001.json').write_text(json.dumps(dict(peak_reserved_bytes=16*2**30,peak_allocated_bytes=1)))
(p/'result.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=1)))
tracker=PeakRecords();self.assertTrue(tracker.scan(p));(p/'history/001.json').unlink();self.assertTrue(tracker.scan(p))
def test_train_does_not_open_test_or_pilot_weights(self):
tree=ast.parse(Path('tools/phase2_h2_s1_worker.py').read_text())
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call)]
self.assertFalse(any(isinstance(n.func,ast.Attribute) and n.func.attr=='load' and isinstance(n.func.value,ast.Name) and n.func.value.id=='torch' for n in calls))
roles=[n.args[3].value for n in calls if isinstance(n.func,ast.Name) and n.func.id=='load_part']
self.assertEqual(sorted(roles),['learning','validation'])
def test_unapproved_scope_rejected_before_sources(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);(p/'contract.json').write_text(json.dumps(dict(approval='wrong',mode='H2_S1_FOLD0_LAST200')))
with self.assertRaises(PermissionError):common.config(p)
if __name__=='__main__':unittest.main()