Freeze approved H2 S1 fold0 last-epoch200 training amendment

This commit is contained in:
Frank14f committed 2026-11-17 16:08:56 +08:00
1 parent a864b29563
commit 7a7174746d
8 files changed
+345

No files matched your search

@@ -0,0 +1,28 @@
{
"status": "PASS",
"tests": 7,
"gpu_used": false,
"science_workload": false,
"setup_cpu_accounting": "included in60s reserve, individual test CPU not measured",
"test_sha256": "7db2e7285e8b441f627b9c1098a31bf26589c92f771f91a9c915d9dadf2cf21f",
"code_sha256": {
"tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81"
}
}
@@ -0,0 +1,73 @@
{
"approval": "1794901608.131369",
"mode": "H2_S1_FOLD0_LAST200",
"pilot_root": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17",
"original_contract_sha256": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
"epochs": 200,
"seed": 0,
"fold": 0,
"code_sha256": {
"tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81"
},
"source_sha256": {
"artifacts/phase2-h1-execution-2026-11-17/contract.json": "416183a09272714a179af790e5b7be85e5a92f1395afa6a5b9fe123a184b44b2",
"artifacts/phase2-h1-execution-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad",
"artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263",
"artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96",
"artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef",
"artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911",
"artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221",
"artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f",
"docs/phase2-h2-architecture-2026-11-17.md": "a15a13319d71b61c7677de41ab5bee2541d77009e0f8a9aab43e7ffccb0ea0c9",
"references/2026-11-17/phase2-design/mold-h2-review-3541052d.md": "3541052da60cf4e79aab1611e2cc66cc4e97c630bdc6bc0b9a69df9bf672a460",
"docs/phase2-h2-execution-contract-2026-11-17.md": "1e74e1c0dcb19bb1c358ec13cb5346111eb3af7a0ef2ca3a11cb549e5da81d96",
"artifacts/phase2-h2-acceptance-2026-11-17/acceptance.json": "c8704f82239c1493a73b160cfa0db5b8ce7af4f3ebe5b164c04a377f6b137c8f",
"artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
"artifacts/phase2-h2-pilot-2026-11-17/complete.json": "b1a2c509354cb0eb0f8a8c8ec9ec46d259b1a2f9a24072ffe6bd547f034cbe60",
"artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
"docs/phase2-h2-s1-amendment-2026-11-17.md": "5c2f4a30faba1eb4edc8316d6fdd6b3172b20cea295a1eefc683075c9b94cf2f"
},
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"worker_cpus": [
0,
1
],
"guard_cpus": [
2
],
"gpu_seconds": 25200,
"cpu_seconds": 25200,
"pilot_setup_charge_seconds": 60,
"total_h2_gpu_seconds": 144000,
"prior_pilot_gpu_seconds": 182.35251634096494,
"prior_pilot_cpu_seconds": 192.51061707999997,
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 21474836480,
"standardization_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json",
"standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
"prediction_gpu_reserve": 600,
"postfit_cpu_cap": 3600,
"checkpoint": "last_epoch200_only",
"no_retry": true,
"s2_gate": "field H2-A CI high<0.005 OR last_distance CI high<0",
"s2_started": false,
"postfit_started": false
}
+11
View File
@@ -0,0 +1,11 @@
# H2-S1 labelled engineering amendment
Claude1794901608.131369 supersedes the pilot's 16 GPU-hour cost gate with staged 40 GPU-hours including the original pilot. Original contract9f616974, PARKED_RESOURCE and all pilot artifacts remain unchanged. This is a resource decision before any formal H2 result, not a model-selection decision.
S1 is one fresh seed0/fold0 fit: original19 features, FP64 F/c skip, full original nodes and edges, width64/four blocks, exact original optimizer and 200 epochs, ten-epoch warmup/cosine. Reuse only learning-side normalization and original learning c, never pilot weights. Validation every10 epochs is diagnostic only; last epoch200 is the sole checkpoint. No early stopping or best-checkpoint selection.
Unique root artifacts/phase2-h2-s1-2026-11-17; single V100 GPU0, workerCPU0–1, guardCPU2. Training cap25200 GPU-seconds includes60 setup and all waits, failures and monitoring; CPU cap25200 seconds includes60 reserve. Original pilot182.352516340965 GPU-seconds is counted once. Prediction allowance<=600 GPU-seconds and CPU postfit<=3600 seconds will be enforced separately before execution, with all GPU charges included in total144000. This initial entry starts training only; it cannot launch predictions, scoring or S2. No retry. Allocated/reserved15GiB, whole treeRSS16GiB, disk20GiB; cumulative JSON history/progress/failure/result peak guard plus device sampling, limits sampled and any final overshoot fails.
After train PASS and exact epoch200 checkpoint freeze, freeze test predictions and scoring source before reading test targets. Compare with the original A on the same fold0 test components and query metrics. S2 folds1–4 are eligible only if paired component field-L2 H2−A CI upper<+0.005 OR last-fill-distance H2−A CI upper<0. Strict bounds, original2000 paired bootstrap/seed, no undefined-bound pass. S2 remains gated, not running. CPU field scoring and topology costs are distinct; this S1 gate does not require new topology computation.
Seven CPU engineering regressions passed: strict OR threshold, final-only checkpoint, reject validation normalization, schedule indexing, historical failure peak retention, no test/pilot-weight read in train, unapproved-scope rejection. Existing model gradient/data acceptance remains unchanged and is not rerun. These are CPU checks, not evidence of actual S1 training. Source closure and inherited inputs are pinned before launch; actual started/worker progress alone establishes execution.
+30
View File
@@ -0,0 +1,30 @@
"""Labelled S1 amendment; original pilot and scientific sources remain immutable."""
import json
from pathlib import Path
import numpy as np
import torch
from phase2_h1_worker_v2 import checked
from phase2_h2_data import config as pilot_config
BASE=Path('/home/frank14f/optane/MeltMind')
def read(p):return json.loads(Path(p).read_text())
def config(root):
c=read(root/'contract.json')
if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
old=pilot_config(Path(c['pilot_root']))
if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed')
if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap')
return c,old
def gate(field_high,last_high):
return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0))
def selected_checkpoint(completed_epochs):
if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated')
return 'last-epoch200.pt'
def stats_from_pilot(c,old):
stats=read(checked(c['standardization_path'],c['standardization_sha256']))
if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch')
if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization')
return stats
+83
View File
@@ -0,0 +1,83 @@
"""Independent bounded S1 train guard; original pilot remains immutable."""
import fcntl
import json
import os
from pathlib import Path
import signal
import sys
import threading
import time
import psutil
from phase2_h2_s1_common import config
from phase2_h1_worker_v2 import save,sha
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def run(root):
c,_=config(root);os.sched_setaffinity(0,c['guard_cpus'])
with (root/'s1.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry')
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)]
def kill_worker():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
def scan():
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT')
for x in requested:reason(x)
save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
if reasons:kill_worker()
done.wait(.5)
thread=None
try:
for i in range(3):
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
if i<2:time.sleep(2)
save(root/'idle-polls.json',samples)
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=c['gpu_seconds']-elapsed()-10
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
for s,handler in old.items():signal.signal(s,handler)
value=None;p=root/'worker/result.json'
if p.exists():value=json.loads(p.read_text())
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(root)
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT')
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors,
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu,
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
+20
View File
@@ -0,0 +1,20 @@
"""Start the one published H2 S1 training after host/GPU duplicate checks."""
import hashlib,json,pathlib,subprocess,sys,time,psutil
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17'
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
contract=json.loads((root/'contract.json').read_text())
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
for p in psutil.process_iter(['pid','cmdline']):
c=p.info['cmdline'] or []
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2'
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
uuid=contract['gpu_uuid']
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
unit='meltmind-phase2-h2-s1-20261117'
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)]
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
print('SERVICE_LAUNCHED',unit,flush=True)
+62
View File
@@ -0,0 +1,62 @@
"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels."""
import json,os,sys,time,traceback
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import seed,peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root):
c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
def record(item,epoch_file=False):
item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stats=stats_from_pilot(c,old);save(out/'standardization.json',stats)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]))
+38
View File
@@ -0,0 +1,38 @@
"""Engineering regressions for the independent S1 scope, no real data/GPU."""
import ast,json,tempfile,unittest
from pathlib import Path
from unittest.mock import patch
import phase2_h2_s1_common as common
from phase2_h2_model import lr_epoch
from s2_main_memory_guard import PeakRecords
class S1Tests(unittest.TestCase):
def test_gate_strict_and_or(self):
self.assertFalse(common.gate(.005,0));self.assertTrue(common.gate(.00499,1));self.assertTrue(common.gate(.1,-1e-10));self.assertFalse(common.gate(None,float('nan')))
def test_final_only(self):
for e in [1,42,199,201]:
with self.assertRaises(ValueError):common.selected_checkpoint(e)
self.assertEqual(common.selected_checkpoint(200),'last-epoch200.pt')
def test_normalization_rejects_validation(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'s.json';p.write_text(json.dumps(dict(parts=['validation'],nodes=1,mean=[0]*19,scale=[1]*19)))
c=dict(standardization_path=str(p),standardization_sha256='dummy')
with patch.object(common,'checked',side_effect=lambda p,h:Path(p)):
with self.assertRaises(ValueError):common.stats_from_pilot(c,dict(folds=[dict(learning=['learning'],learning_nodes=1)]))
def test_schedule_index(self):
self.assertAlmostEqual(lr_epoch(1),.0001);self.assertAlmostEqual(lr_epoch(10),.001);self.assertAlmostEqual(lr_epoch(200),.00001);self.assertLess(lr_epoch(11),lr_epoch(10))
def test_history_peak_cannot_be_hidden(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);(p/'history').mkdir();(p/'history/001.json').write_text(json.dumps(dict(peak_reserved_bytes=16*2**30,peak_allocated_bytes=1)))
(p/'result.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=1)))
tracker=PeakRecords();self.assertTrue(tracker.scan(p));(p/'history/001.json').unlink();self.assertTrue(tracker.scan(p))
def test_train_does_not_open_test_or_pilot_weights(self):
tree=ast.parse(Path('tools/phase2_h2_s1_worker.py').read_text())
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call)]
self.assertFalse(any(isinstance(n.func,ast.Attribute) and n.func.attr=='load' and isinstance(n.func.value,ast.Name) and n.func.value.id=='torch' for n in calls))
roles=[n.args[3].value for n in calls if isinstance(n.func,ast.Name) and n.func.id=='load_part']
self.assertEqual(sorted(roles),['learning','validation'])
def test_unapproved_scope_rejected_before_sources(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);(p/'contract.json').write_text(json.dumps(dict(approval='wrong',mode='H2_S1_FOLD0_LAST200')))
with self.assertRaises(PermissionError):common.config(p)
if __name__=='__main__':unittest.main()