diff --git a/artifacts/phase2-h2-s1-2026-11-17/acceptance.json b/artifacts/phase2-h2-s1-2026-11-17/acceptance.json new file mode 100644 index 000000000..f018a64d6 --- /dev/null +++ b/artifacts/phase2-h2-s1-2026-11-17/acceptance.json @@ -0,0 +1,28 @@ +{ + "status": "PASS", + "tests": 7, + "gpu_used": false, + "science_workload": false, + "setup_cpu_accounting": "included in60s reserve, individual test CPU not measured", + "test_sha256": "7db2e7285e8b441f627b9c1098a31bf26589c92f771f91a9c915d9dadf2cf21f", + "code_sha256": { + "tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81" + } +} diff --git a/artifacts/phase2-h2-s1-2026-11-17/contract.json b/artifacts/phase2-h2-s1-2026-11-17/contract.json new file mode 100644 index 000000000..ca5f09045 --- /dev/null +++ b/artifacts/phase2-h2-s1-2026-11-17/contract.json @@ -0,0 +1,73 @@ +{ + "approval": "1794901608.131369", + "mode": "H2_S1_FOLD0_LAST200", + "pilot_root": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17", + "original_contract_sha256": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b", + "epochs": 200, + "seed": 0, + "fold": 0, + "code_sha256": { + "tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81" + }, + "source_sha256": { + "artifacts/phase2-h1-execution-2026-11-17/contract.json": "416183a09272714a179af790e5b7be85e5a92f1395afa6a5b9fe123a184b44b2", + "artifacts/phase2-h1-execution-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad", + "artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263", + "artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef", + "artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911", + "artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221", + "artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f", + "docs/phase2-h2-architecture-2026-11-17.md": "a15a13319d71b61c7677de41ab5bee2541d77009e0f8a9aab43e7ffccb0ea0c9", + "references/2026-11-17/phase2-design/mold-h2-review-3541052d.md": "3541052da60cf4e79aab1611e2cc66cc4e97c630bdc6bc0b9a69df9bf672a460", + "docs/phase2-h2-execution-contract-2026-11-17.md": "1e74e1c0dcb19bb1c358ec13cb5346111eb3af7a0ef2ca3a11cb549e5da81d96", + "artifacts/phase2-h2-acceptance-2026-11-17/acceptance.json": "c8704f82239c1493a73b160cfa0db5b8ce7af4f3ebe5b164c04a377f6b137c8f", + "artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b", + "artifacts/phase2-h2-pilot-2026-11-17/complete.json": "b1a2c509354cb0eb0f8a8c8ec9ec46d259b1a2f9a24072ffe6bd547f034cbe60", + "artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "docs/phase2-h2-s1-amendment-2026-11-17.md": "5c2f4a30faba1eb4edc8316d6fdd6b3172b20cea295a1eefc683075c9b94cf2f" + }, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "worker_cpus": [ + 0, + 1 + ], + "guard_cpus": [ + 2 + ], + "gpu_seconds": 25200, + "cpu_seconds": 25200, + "pilot_setup_charge_seconds": 60, + "total_h2_gpu_seconds": 144000, + "prior_pilot_gpu_seconds": 182.35251634096494, + "prior_pilot_cpu_seconds": 192.51061707999997, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480, + "standardization_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json", + "standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "prediction_gpu_reserve": 600, + "postfit_cpu_cap": 3600, + "checkpoint": "last_epoch200_only", + "no_retry": true, + "s2_gate": "field H2-A CI high<0.005 OR last_distance CI high<0", + "s2_started": false, + "postfit_started": false +} diff --git a/docs/phase2-h2-s1-amendment-2026-11-17.md b/docs/phase2-h2-s1-amendment-2026-11-17.md new file mode 100644 index 000000000..f934504ab --- /dev/null +++ b/docs/phase2-h2-s1-amendment-2026-11-17.md @@ -0,0 +1,11 @@ +# H2-S1 labelled engineering amendment + +Claude1794901608.131369 supersedes the pilot's 16 GPU-hour cost gate with staged 40 GPU-hours including the original pilot. Original contract9f616974, PARKED_RESOURCE and all pilot artifacts remain unchanged. This is a resource decision before any formal H2 result, not a model-selection decision. + +S1 is one fresh seed0/fold0 fit: original19 features, FP64 F/c skip, full original nodes and edges, width64/four blocks, exact original optimizer and 200 epochs, ten-epoch warmup/cosine. Reuse only learning-side normalization and original learning c, never pilot weights. Validation every10 epochs is diagnostic only; last epoch200 is the sole checkpoint. No early stopping or best-checkpoint selection. + +Unique root artifacts/phase2-h2-s1-2026-11-17; single V100 GPU0, workerCPU0–1, guardCPU2. Training cap25200 GPU-seconds includes60 setup and all waits, failures and monitoring; CPU cap25200 seconds includes60 reserve. Original pilot182.352516340965 GPU-seconds is counted once. Prediction allowance<=600 GPU-seconds and CPU postfit<=3600 seconds will be enforced separately before execution, with all GPU charges included in total144000. This initial entry starts training only; it cannot launch predictions, scoring or S2. No retry. Allocated/reserved15GiB, whole treeRSS16GiB, disk20GiB; cumulative JSON history/progress/failure/result peak guard plus device sampling, limits sampled and any final overshoot fails. + +After train PASS and exact epoch200 checkpoint freeze, freeze test predictions and scoring source before reading test targets. Compare with the original A on the same fold0 test components and query metrics. S2 folds1–4 are eligible only if paired component field-L2 H2−A CI upper<+0.005 OR last-fill-distance H2−A CI upper<0. Strict bounds, original2000 paired bootstrap/seed, no undefined-bound pass. S2 remains gated, not running. CPU field scoring and topology costs are distinct; this S1 gate does not require new topology computation. + +Seven CPU engineering regressions passed: strict OR threshold, final-only checkpoint, reject validation normalization, schedule indexing, historical failure peak retention, no test/pilot-weight read in train, unapproved-scope rejection. Existing model gradient/data acceptance remains unchanged and is not rerun. These are CPU checks, not evidence of actual S1 training. Source closure and inherited inputs are pinned before launch; actual started/worker progress alone establishes execution. diff --git a/tools/phase2_h2_s1_common.py b/tools/phase2_h2_s1_common.py new file mode 100644 index 000000000..396050685 --- /dev/null +++ b/tools/phase2_h2_s1_common.py @@ -0,0 +1,30 @@ +"""Labelled S1 amendment; original pilot and scientific sources remain immutable.""" +import json +from pathlib import Path +import numpy as np +import torch +from phase2_h1_worker_v2 import checked +from phase2_h2_data import config as pilot_config +BASE=Path('/home/frank14f/optane/MeltMind') +def read(p):return json.loads(Path(p).read_text()) +def config(root): + c=read(root/'contract.json') + if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope') + for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h) + old=pilot_config(Path(c['pilot_root'])) + if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed') + if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap') + return c,old + +def gate(field_high,last_high): + return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0)) + +def selected_checkpoint(completed_epochs): + if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated') + return 'last-epoch200.pt' + +def stats_from_pilot(c,old): + stats=read(checked(c['standardization_path'],c['standardization_sha256'])) + if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch') + if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization') + return stats diff --git a/tools/phase2_h2_s1_guard.py b/tools/phase2_h2_s1_guard.py new file mode 100644 index 000000000..65b409d62 --- /dev/null +++ b/tools/phase2_h2_s1_guard.py @@ -0,0 +1,83 @@ +"""Independent bounded S1 train guard; original pilot remains immutable.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase2_h2_s1_common import config +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(root): + c,_=config(root);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'s1.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds'])) + def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + if p.exists():value=json.loads(p.read_text()) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(root) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]))) diff --git a/tools/phase2_h2_s1_launch.py b/tools/phase2_h2_s1_launch.py new file mode 100644 index 000000000..46926da14 --- /dev/null +++ b/tools/phase2_h2_s1_launch.py @@ -0,0 +1,20 @@ +"""Start the one published H2 S1 training after host/GPU duplicate checks.""" +import hashlib,json,pathlib,subprocess,sys,time,psutil +B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17' +proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published'] +for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p +contract=json.loads((root/'contract.json').read_text()) +for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p +for p in psutil.process_iter(['pid','cmdline']): + c=p.info['cmdline'] or [] + assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2' +assert not (root/'started.json').exists() and not (root/'service-launch.json').exists() +uuid=contract['gpu_uuid'] +raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True) +mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0 +unit='meltmind-phase2-h2-s1-20261117' +cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B), + '--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)] +r=subprocess.run(cmd,check=True,capture_output=True,text=True) +(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n') +print('SERVICE_LAUNCHED',unit,flush=True) diff --git a/tools/phase2_h2_s1_worker.py b/tools/phase2_h2_s1_worker.py new file mode 100644 index 000000000..f62aa6c24 --- /dev/null +++ b/tools/phase2_h2_s1_worker.py @@ -0,0 +1,62 @@ +"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels.""" +import json,os,sys,time,traceback +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import seed,peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root): + c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stats=stats_from_pilot(c,old);save(out/'standardization.json',stats) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1])) diff --git a/tools/test_phase2_h2_s1.py b/tools/test_phase2_h2_s1.py new file mode 100644 index 000000000..d896c7e40 --- /dev/null +++ b/tools/test_phase2_h2_s1.py @@ -0,0 +1,38 @@ +"""Engineering regressions for the independent S1 scope, no real data/GPU.""" +import ast,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +import phase2_h2_s1_common as common +from phase2_h2_model import lr_epoch +from s2_main_memory_guard import PeakRecords +class S1Tests(unittest.TestCase): + def test_gate_strict_and_or(self): + self.assertFalse(common.gate(.005,0));self.assertTrue(common.gate(.00499,1));self.assertTrue(common.gate(.1,-1e-10));self.assertFalse(common.gate(None,float('nan'))) + def test_final_only(self): + for e in [1,42,199,201]: + with self.assertRaises(ValueError):common.selected_checkpoint(e) + self.assertEqual(common.selected_checkpoint(200),'last-epoch200.pt') + def test_normalization_rejects_validation(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'s.json';p.write_text(json.dumps(dict(parts=['validation'],nodes=1,mean=[0]*19,scale=[1]*19))) + c=dict(standardization_path=str(p),standardization_sha256='dummy') + with patch.object(common,'checked',side_effect=lambda p,h:Path(p)): + with self.assertRaises(ValueError):common.stats_from_pilot(c,dict(folds=[dict(learning=['learning'],learning_nodes=1)])) + def test_schedule_index(self): + self.assertAlmostEqual(lr_epoch(1),.0001);self.assertAlmostEqual(lr_epoch(10),.001);self.assertAlmostEqual(lr_epoch(200),.00001);self.assertLess(lr_epoch(11),lr_epoch(10)) + def test_history_peak_cannot_be_hidden(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'history').mkdir();(p/'history/001.json').write_text(json.dumps(dict(peak_reserved_bytes=16*2**30,peak_allocated_bytes=1))) + (p/'result.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=1))) + tracker=PeakRecords();self.assertTrue(tracker.scan(p));(p/'history/001.json').unlink();self.assertTrue(tracker.scan(p)) + def test_train_does_not_open_test_or_pilot_weights(self): + tree=ast.parse(Path('tools/phase2_h2_s1_worker.py').read_text()) + calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call)] + self.assertFalse(any(isinstance(n.func,ast.Attribute) and n.func.attr=='load' and isinstance(n.func.value,ast.Name) and n.func.value.id=='torch' for n in calls)) + roles=[n.args[3].value for n in calls if isinstance(n.func,ast.Name) and n.func.id=='load_part'] + self.assertEqual(sorted(roles),['learning','validation']) + def test_unapproved_scope_rejected_before_sources(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'contract.json').write_text(json.dumps(dict(approval='wrong',mode='H2_S1_FOLD0_LAST200'))) + with self.assertRaises(PermissionError):common.config(p) +if __name__=='__main__':unittest.main()