Freeze reviewed H2 pilot contract data guards and cost projection
This commit is contained in:
1 parent
00ec7cacf8
commit
6c42bfb1ce
21 files changed
+20668
No files matched your search
@@ -0,0 +1,87 @@
|
||||
"""Frozen H1 interface and label-scoped H2 inputs; no test target access."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from scipy.sparse import csr_matrix
|
||||
from scipy.sparse.csgraph import connected_components
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase2_h2_model import edge_attributes
|
||||
from phase2_residual_math import residual_target
|
||||
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794900702.867059' or c['mode']!='PILOT_ONLY':raise PermissionError('No approved pilot contract')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
||||
if torch.__version__!=c['versions']['torch'] or np.__version__!=c['versions']['numpy']:raise ValueError('Library identity changed')
|
||||
return c
|
||||
|
||||
def cached(c,case,name):
|
||||
r=c['parts'][case];return np.load(checked(Path(r['output'])/(name+'.npy'),r['output_sha256'][name+'.npy']),allow_pickle=False)
|
||||
|
||||
def standardization(c,root):
|
||||
n=0;mean=np.zeros(19,np.float64);M2=np.zeros(19,np.float64);seen=[];start=time.process_time()
|
||||
for case in c['folds'][0]['learning']:
|
||||
x=np.asarray(cached(c,case,'features'),np.float64)
|
||||
if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid')
|
||||
k=len(x);avg=x.mean(0);delta=avg-mean;batch=np.square(x-avg).sum(0)
|
||||
M2+=batch+delta*delta*(n*k/(n+k));mean+=delta*(k/(n+k));n+=k;seen.append(case)
|
||||
save(root/'progress.json',dict(stage='LEARNING_ONLY_STANDARDIZATION',parts=len(seen),nodes=n,cpu_seconds=time.process_time()-start,labels_read=False))
|
||||
std=np.sqrt(M2/n);scale=np.where(std==0,1.,std)
|
||||
if not np.isfinite(scale).all() or n!=c['folds'][0]['learning_nodes']:raise ValueError('Standardization coverage')
|
||||
out=dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),nodes=n,parts=seen,labels_read=False,cpu_seconds=time.process_time()-start)
|
||||
save(root/'standardization.json',out);return out
|
||||
|
||||
def load_part(c,case,stats,role,root):
|
||||
allowed=c['folds'][0][role] if role in ('learning','validation') else []
|
||||
if case not in allowed:raise PermissionError('Pilot role violation')
|
||||
r=c['parts'][case];g=c['geometry'][case]
|
||||
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:
|
||||
points=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy()
|
||||
indptr=z['indptr'].copy();sender=z['indices'].astype(np.int64);old_rg=float(z['rg'])
|
||||
n=len(raw);receiver=np.repeat(np.arange(n,dtype=np.int64),np.diff(indptr))
|
||||
if not np.array_equal(raw,cached(c,case,'raw_ids')) or not np.array_equal(mass,cached(c,case,'mass')):raise ValueError('Frozen geometry/node mass identity')
|
||||
if n!=r['nodes'] or len(sender)!=g['directed_edges']:raise ValueError('Frozen graph counts')
|
||||
a,rg=edge_attributes(points,mass,receiver,sender)
|
||||
if not np.isclose(rg,old_rg,rtol=1e-10,atol=1e-12):raise ValueError('Original mass Rg differs')
|
||||
graph=csr_matrix((np.ones(len(sender)),sender,indptr),shape=(n,n));count,comp=connected_components(graph,directed=False)
|
||||
x=cached(c,case,'features');F=cached(c,case,'Vf')
|
||||
if F.dtype!=np.float64 or F.shape!=(n,) or not np.isfinite(F).all():raise ValueError('Original F must be FP64')
|
||||
x=((np.asarray(x,np.float64)-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
||||
if not np.isfinite(x).all():raise ValueError('Standardized feature overflow')
|
||||
result=dict(x=torch.from_numpy(x),mass=torch.from_numpy(mass.astype(np.float32)),receiver=torch.from_numpy(receiver),sender=torch.from_numpy(sender),attributes=torch.from_numpy(a),component_ids=torch.from_numpy(comp.astype(np.int64)),
|
||||
F=F,raw_ids=raw,physical_components=int(count),nodes=n,directed_edges=len(sender),Rg=rg,t_set=r['t_set'])
|
||||
if role=='learning':
|
||||
# Append the intended role/hash before opening any cached target.
|
||||
with (root/'learning-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,role=role,target_sha256=r['output_sha256']['target.npy'],clock_sha256=c['folds'][0]['clock_sha256']))+'\n')
|
||||
y=cached(c,case,'target');result['target']=torch.from_numpy(residual_target(y,F,c['folds'][0]['c'],fit_dtype='float32'))
|
||||
return result
|
||||
|
||||
def to_device(data,device):
|
||||
return {k:(v.to(device) if isinstance(v,torch.Tensor) else v) for k,v in data.items()}
|
||||
|
||||
def arguments(data):return tuple(data[k] for k in ['x','mass','receiver','sender','attributes','component_ids'])
|
||||
|
||||
def projection(c,epoch_seconds,validation_seconds,pilot_seconds):
|
||||
f0=c['folds'][0];per_fold=[]
|
||||
for f in c['folds']:
|
||||
learning_ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],len(f['learning'])/len(f0['learning']))
|
||||
validation_ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation']))
|
||||
prediction_ratio=max(f['test_nodes']/f0['validation_nodes'],f['test_edges']/f0['validation_edges'],len(f['test'])/len(f0['validation']))
|
||||
per_fold.append(dict(fold=f['fold'],train_200_seconds=200*epoch_seconds*learning_ratio,validation_20_seconds=20*validation_seconds*validation_ratio,test_prediction_seconds=validation_seconds*prediction_ratio,
|
||||
learning_ratio=learning_ratio,validation_ratio=validation_ratio,test_ratio=prediction_ratio))
|
||||
# Every measured forward/backward time includes load, transfers and optimizer.
|
||||
training=1.25*sum(f['train_200_seconds']+f['validation_20_seconds'] for f in per_fold)
|
||||
prediction=1.25*sum(f['test_prediction_seconds'] for f in per_fold)
|
||||
total=pilot_seconds+training+prediction+c['future_setup_reserve_seconds']
|
||||
passed=training<=12*3600 and prediction<=2*3600 and total<=16*3600
|
||||
return dict(per_fold=per_fold,measured_full_epoch_seconds=epoch_seconds,measured_validation_inference_seconds=validation_seconds,
|
||||
safety_factor=1.25,training_with_validation_seconds=training,prediction_seconds=prediction,pilot_seconds=pilot_seconds,future_setup_seconds=c['future_setup_reserve_seconds'],total_gpu_seconds=total,
|
||||
fits_12h_training_2h_prediction_16h_total=passed,decision='COST_FEASIBLE_FULL_SCOPE_AUTHORIZED' if passed else 'PARKED_RESOURCE',
|
||||
full_run_authorized_by='1794900702.867059',full_run_started=False,epoch_target=200,folds=5,new_fits=5,validation_targets_read=False,
|
||||
limitation='Validation forward timing only; no target metrics in pilot. CPU final field/topology scoring is unmeasured and cannot be claimed affordable from this GPU projection.')
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Unique H2 pilot; includes setup/idle, recorded failure peaks and final CPU."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h2_data import config,projection
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(root):
|
||||
c=config(root);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'pilot.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single pilot launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
|
||||
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase2_h2_pilot_worker.py',str(root)]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('PILOT_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='PILOT_RUNNING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','pilot',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_PILOT_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
projected=None
|
||||
if value:
|
||||
projected=projection(c,value['full_epoch_seconds'],value['validation_seconds'],elapsed());save(root/'projection.json',projected)
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision=projected['decision'] if projected and not reasons else 'PILOT_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cpu_seconds_charged=totalcpu,projection=projected,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,full_run_started=False,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Start the one published H2 pilot after host/GPU duplicate checks."""
|
||||
import hashlib,json,pathlib,subprocess,sys,time,psutil
|
||||
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-pilot-2026-11-17'
|
||||
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
|
||||
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
contract=json.loads((root/'contract.json').read_text())
|
||||
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
c=p.info['cmdline'] or []
|
||||
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py'}.intersection(c),'Existing H2'
|
||||
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
|
||||
uuid=contract['gpu_uuid']
|
||||
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
|
||||
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
|
||||
unit='meltmind-phase2-h2-pilot-20261117'
|
||||
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
|
||||
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_guard.py',str(root)]
|
||||
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
|
||||
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
|
||||
print('SERVICE_LAUNCHED',unit,flush=True)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing."""
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection
|
||||
from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_residual_math import decode_seconds
|
||||
|
||||
def peaks():
|
||||
return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
|
||||
def seed():
|
||||
random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
|
||||
def train_step(model,opt,data,parts,total):
|
||||
model.train();opt.zero_grad(set_to_none=True)
|
||||
prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total)
|
||||
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
|
||||
loss.backward()
|
||||
norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
|
||||
opt.step()
|
||||
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
|
||||
torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks())
|
||||
|
||||
def run(root):
|
||||
c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0
|
||||
(out/'history').mkdir()
|
||||
def record(value):
|
||||
nonlocal history_index
|
||||
item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value)
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'history'/f'{history_index:05d}.json',item);history_index+=1
|
||||
save(out/'progress.json',item);return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs')
|
||||
stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
props=torch.cuda.get_device_properties(0)
|
||||
save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0))))
|
||||
fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD'
|
||||
for case in c['representatives']:
|
||||
seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize()
|
||||
for step in range(2):
|
||||
tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r))
|
||||
records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks()))
|
||||
del data,opt,model;gc.collect();torch.cuda.empty_cache()
|
||||
save(out/'representative-timings.json',records)
|
||||
stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[]
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total)
|
||||
trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data
|
||||
torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start
|
||||
if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch')
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(2)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt')
|
||||
save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt')))
|
||||
stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize()
|
||||
validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds
|
||||
duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start
|
||||
projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected)
|
||||
save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Metadata-only pilot contract from frozen H1 inputs; never accesses labels."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import sha,save,checked
|
||||
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
|
||||
def closure(names):
|
||||
pending=list(names);done={}
|
||||
while pending:
|
||||
name=pending.pop();path=BASE/'tools'/Path(*name.split('.')).with_suffix('.py')
|
||||
if not path.exists() or name in done:continue
|
||||
done[name]=path
|
||||
for node in ast.walk(ast.parse(path.read_text())):
|
||||
if isinstance(node,ast.Import):pending.extend(a.name for a in node.names)
|
||||
elif isinstance(node,ast.ImportFrom) and node.module:pending.append(node.module)
|
||||
return {str(p.relative_to(BASE)):sha(p) for p in done.values()}
|
||||
|
||||
def run(acceptance):
|
||||
acc=json.loads(acceptance.read_text())
|
||||
if acc['status']!='PASS':raise ValueError('Final acceptance not PASS')
|
||||
for p,h in acc['code_sha256'].items():checked(BASE/p,h)
|
||||
oldroot=BASE/'artifacts/phase2-h1-execution-2026-11-17';sources={}
|
||||
def read(p):
|
||||
p=Path(p);sources[str(p.relative_to(BASE))]=sha(p);return json.loads(p.read_text())
|
||||
old=read(oldroot/'contract.json');models=read(oldroot/'models-terminal.json')['models'];gm=read(oldroot/'geometry/manifest.json')
|
||||
geometry={r['case']:r for r in gm['records'] if r['protocol']=='grouped'};folds=[]
|
||||
for fold in range(5):
|
||||
job=next(j for j in old['jobs'] if j['protocol']=='grouped' and j['pool']==100 and j['fold']==fold)
|
||||
model=next(m for m in models if m['index']==job['index']);assert model['status']=='PASS';checked(model['clock_path'],model['clock_sha256']);clock=read(model['clock_path']);assert clock['parts']==job['learning']
|
||||
rows=old['grouped_rows'];validation=sorted(case for case,r in rows.items() if next(a for a in r['assignments'] if a['fold']==fold)['role']=='validation')
|
||||
f=dict(fold=fold,learning=job['learning'],validation=validation,test=job['test'],c=clock['c'],clock_sha256=model['clock_sha256'],clock_path=model['clock_path'])
|
||||
for role in ['learning','validation','test']:
|
||||
f[role+'_nodes']=sum(geometry[case]['nodes'] for case in f[role]);f[role+'_edges']=sum(geometry[case]['directed_edges'] for case in f[role])
|
||||
assert not(set(f['learning'])&set(f['validation']) or set(f['learning'])&set(f['test']) or set(f['validation'])&set(f['test']))
|
||||
folds.append(f)
|
||||
reps=[]
|
||||
for key in ['nodes','directed_edges']:
|
||||
order=sorted(folds[0]['learning'],key=lambda case:(geometry[case][key],case))
|
||||
for i in [0,len(order)//2,len(order)-1]:
|
||||
if order[i] not in reps:reps.append(order[i])
|
||||
for rel in ['docs/phase2-h2-architecture-2026-11-17.md','references/2026-11-17/phase2-design/mold-h2-review-3541052d.md','docs/phase2-h2-execution-contract-2026-11-17.md']:
|
||||
sources[rel]=sha(BASE/rel)
|
||||
sources[str(acceptance.relative_to(BASE))]=sha(acceptance)
|
||||
root=BASE/'artifacts/phase2-h2-pilot-2026-11-17';root.mkdir(exist_ok=False)
|
||||
names=['phase2_h2_model','phase2_h2_data','phase2_h2_pilot_worker','phase2_h2_guard','phase2_h2_prepare']
|
||||
save(root/'contract.json',dict(version='phase2-h2-pilot-v1',approval='1794900702.867059',initial_pilot_approval='1794900338.137689',mode='PILOT_ONLY',full_after_cost_gate_authorized=True,
|
||||
parts=old['parts']['grouped'],geometry=geometry,folds=folds,representatives=reps,code_sha256=closure(names),source_sha256=sources,
|
||||
gpu_uuid='GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',gpu_index=0,worker_cpus=[0,1],guard_cpus=[2],gpu_seconds=7200,cpu_seconds=7200,pilot_setup_charge_seconds=60,future_setup_reserve_seconds=300,
|
||||
rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30,versions=dict(torch=torch.__version__,numpy=np.__version__,cuda_build=torch.version.cuda),
|
||||
model=dict(width=64,blocks=4,tokens_per_physical_component=64,attention_heads=4,edge_chunk=65536,node_chunk=8192,activation_checkpoint='each full block, non-reentrant',dtype='FP32',dropout=0),
|
||||
pilot='learning-only normalization; small/median/max node and edge representatives two steps each; fresh seed0 complete fold0 epoch; complete validation forward timing with no targets',
|
||||
optimizer='AdamW lr1e-3 betas.9/.999 eps1e-8 weight_decay1e-4 except biases/norm; clip1; one full-part step weighted Nparts*Ni/Ntotal',
|
||||
schedule='1-based e1..10:1e-4+(e-1)*9e-4/9; e11..200:1e-5+(.001-.00001)*(1+cos(pi*(e-10)/190))/2; step after complete epoch',
|
||||
normalization='learning-only original19 float64 parallel-Welford population std; zero std=>1; standardized network FP32; separate originalFP64 Vf skip',
|
||||
limits_sampled=True,no_new_decodes=True,no_test_targets=True,no_retry=True,full_epochs=200,full_folds=5,full_seed=0,full_gpu_seconds_including_pilot=57600,full_max_parallel_gpus=4))
|
||||
print(json.dumps(dict(root=str(root),contract_sha256=sha(root/'contract.json'),representatives=reps,code_files=len(closure(names)),fold0_learning=len(folds[0]['learning']),validation=len(folds[0]['validation']))))
|
||||
|
||||
if __name__=='__main__':run(Path(sys.argv[1]).resolve())
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Synthetic data-role, moment and conservative-cost contract checks."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
from phase2_h2_data import standardization,load_part,projection
|
||||
|
||||
class Integration(unittest.TestCase):
|
||||
def test_history_only_peak_survives_latest_progress_regression(self):
|
||||
from s2_main_memory_guard import PeakRecords,VRAM_BYTES
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
r=Path(temp);(r/'history').mkdir()
|
||||
(r/'history/0000.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=VRAM_BYTES+1,peak_allocated_bytes=1)))
|
||||
(r/'progress.json').write_text(json.dumps(dict(peak_reserved_bytes=1,peak_allocated_bytes=1)))
|
||||
tracker=PeakRecords();self.assertTrue(tracker.scan(r))
|
||||
self.assertTrue(tracker.witnesses['peak_reserved_bytes']['path'].endswith('history/0000.json'))
|
||||
def test_learning_only_welford_population(self):
|
||||
values={'a':np.arange(57,dtype=np.float64).reshape(3,19),'b':np.arange(95,dtype=np.float64).reshape(5,19)+.3}
|
||||
for x in values.values():x[:,2]=4
|
||||
c=dict(folds=[dict(learning=['a','b'],learning_nodes=8)],parts={'a':dict(nodes=3),'b':dict(nodes=5)})
|
||||
seen=[]
|
||||
def cache(c,case,name):
|
||||
self.assertEqual(name,'features');seen.append(case);return values[case]
|
||||
with tempfile.TemporaryDirectory() as t,patch('phase2_h2_data.cached',side_effect=cache):
|
||||
out=standardization(c,Path(t));allx=np.concatenate(list(values.values()))
|
||||
np.testing.assert_allclose(out['mean'],allx.mean(0));np.testing.assert_allclose(out['population_std'],allx.std(0),atol=1e-13)
|
||||
self.assertEqual(out['scale'][2],1.);self.assertEqual(seen,['a','b']);self.assertFalse(out['labels_read'])
|
||||
def test_test_role_refused_before_open(self):
|
||||
c=dict(folds=[dict(learning=['a'],validation=['b'])])
|
||||
with patch('phase2_h2_data.cached',side_effect=AssertionError('must not read')):
|
||||
with self.assertRaises(PermissionError):load_part(c,'sealed',{},'test',Path('/nonexistent'))
|
||||
with self.assertRaises(PermissionError):load_part(c,'b',{},'learning',Path('/nonexistent'))
|
||||
def test_projection_all_five_folds_and_pilot_once(self):
|
||||
f=dict(fold=0,learning=['a']*2,validation=['b'],test=['c'],learning_nodes=20,learning_edges=100,validation_nodes=10,validation_edges=50,test_nodes=10,test_edges=50)
|
||||
c=dict(folds=[dict(f,fold=i) for i in range(5)],future_setup_reserve_seconds=300)
|
||||
p=projection(c,20,1,600);self.assertEqual(len(p['per_fold']),5)
|
||||
self.assertAlmostEqual(p['training_with_validation_seconds'],1.25*5*(200*20+20));self.assertAlmostEqual(p['total_gpu_seconds'],600+300+1.25*5*(4000+20+1))
|
||||
self.assertTrue(p['fits_12h_training_2h_prediction_16h_total']);self.assertFalse(projection(c,100,1,600)['fits_12h_training_2h_prediction_16h_total'])
|
||||
def test_projection_uses_slowest_node_edge_part_scaling(self):
|
||||
f=dict(fold=0,learning=['a'],validation=['b'],test=['c'],learning_nodes=20,learning_edges=100,validation_nodes=10,validation_edges=50,test_nodes=10,test_edges=50)
|
||||
other=dict(f,fold=1,learning_nodes=40,learning_edges=400)
|
||||
p=projection(dict(folds=[f,other],future_setup_reserve_seconds=300),1,1,60)
|
||||
self.assertEqual(p['per_fold'][1]['learning_ratio'],4)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user