Freeze approved H2 S1 fold0 last-epoch200 training amendment
This commit is contained in:
1 parent
a864b29563
commit
7a7174746d
8 files changed
+345
No files matched your search
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 7,
|
||||
"gpu_used": false,
|
||||
"science_workload": false,
|
||||
"setup_cpu_accounting": "included in60s reserve, individual test CPU not measured",
|
||||
"test_sha256": "7db2e7285e8b441f627b9c1098a31bf26589c92f771f91a9c915d9dadf2cf21f",
|
||||
"code_sha256": {
|
||||
"tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"approval": "1794901608.131369",
|
||||
"mode": "H2_S1_FOLD0_LAST200",
|
||||
"pilot_root": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17",
|
||||
"original_contract_sha256": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
|
||||
"epochs": 200,
|
||||
"seed": 0,
|
||||
"fold": 0,
|
||||
"code_sha256": {
|
||||
"tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81"
|
||||
},
|
||||
"source_sha256": {
|
||||
"artifacts/phase2-h1-execution-2026-11-17/contract.json": "416183a09272714a179af790e5b7be85e5a92f1395afa6a5b9fe123a184b44b2",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221",
|
||||
"artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f",
|
||||
"docs/phase2-h2-architecture-2026-11-17.md": "a15a13319d71b61c7677de41ab5bee2541d77009e0f8a9aab43e7ffccb0ea0c9",
|
||||
"references/2026-11-17/phase2-design/mold-h2-review-3541052d.md": "3541052da60cf4e79aab1611e2cc66cc4e97c630bdc6bc0b9a69df9bf672a460",
|
||||
"docs/phase2-h2-execution-contract-2026-11-17.md": "1e74e1c0dcb19bb1c358ec13cb5346111eb3af7a0ef2ca3a11cb549e5da81d96",
|
||||
"artifacts/phase2-h2-acceptance-2026-11-17/acceptance.json": "c8704f82239c1493a73b160cfa0db5b8ce7af4f3ebe5b164c04a377f6b137c8f",
|
||||
"artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
|
||||
"artifacts/phase2-h2-pilot-2026-11-17/complete.json": "b1a2c509354cb0eb0f8a8c8ec9ec46d259b1a2f9a24072ffe6bd547f034cbe60",
|
||||
"artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
|
||||
"docs/phase2-h2-s1-amendment-2026-11-17.md": "5c2f4a30faba1eb4edc8316d6fdd6b3172b20cea295a1eefc683075c9b94cf2f"
|
||||
},
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"worker_cpus": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"guard_cpus": [
|
||||
2
|
||||
],
|
||||
"gpu_seconds": 25200,
|
||||
"cpu_seconds": 25200,
|
||||
"pilot_setup_charge_seconds": 60,
|
||||
"total_h2_gpu_seconds": 144000,
|
||||
"prior_pilot_gpu_seconds": 182.35251634096494,
|
||||
"prior_pilot_cpu_seconds": 192.51061707999997,
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"disk_bytes": 21474836480,
|
||||
"standardization_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json",
|
||||
"standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
|
||||
"prediction_gpu_reserve": 600,
|
||||
"postfit_cpu_cap": 3600,
|
||||
"checkpoint": "last_epoch200_only",
|
||||
"no_retry": true,
|
||||
"s2_gate": "field H2-A CI high<0.005 OR last_distance CI high<0",
|
||||
"s2_started": false,
|
||||
"postfit_started": false
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# H2-S1 labelled engineering amendment
|
||||
|
||||
Claude1794901608.131369 supersedes the pilot's 16 GPU-hour cost gate with staged 40 GPU-hours including the original pilot. Original contract9f616974, PARKED_RESOURCE and all pilot artifacts remain unchanged. This is a resource decision before any formal H2 result, not a model-selection decision.
|
||||
|
||||
S1 is one fresh seed0/fold0 fit: original19 features, FP64 F/c skip, full original nodes and edges, width64/four blocks, exact original optimizer and 200 epochs, ten-epoch warmup/cosine. Reuse only learning-side normalization and original learning c, never pilot weights. Validation every10 epochs is diagnostic only; last epoch200 is the sole checkpoint. No early stopping or best-checkpoint selection.
|
||||
|
||||
Unique root artifacts/phase2-h2-s1-2026-11-17; single V100 GPU0, workerCPU0–1, guardCPU2. Training cap25200 GPU-seconds includes60 setup and all waits, failures and monitoring; CPU cap25200 seconds includes60 reserve. Original pilot182.352516340965 GPU-seconds is counted once. Prediction allowance<=600 GPU-seconds and CPU postfit<=3600 seconds will be enforced separately before execution, with all GPU charges included in total144000. This initial entry starts training only; it cannot launch predictions, scoring or S2. No retry. Allocated/reserved15GiB, whole treeRSS16GiB, disk20GiB; cumulative JSON history/progress/failure/result peak guard plus device sampling, limits sampled and any final overshoot fails.
|
||||
|
||||
After train PASS and exact epoch200 checkpoint freeze, freeze test predictions and scoring source before reading test targets. Compare with the original A on the same fold0 test components and query metrics. S2 folds1–4 are eligible only if paired component field-L2 H2−A CI upper<+0.005 OR last-fill-distance H2−A CI upper<0. Strict bounds, original2000 paired bootstrap/seed, no undefined-bound pass. S2 remains gated, not running. CPU field scoring and topology costs are distinct; this S1 gate does not require new topology computation.
|
||||
|
||||
Seven CPU engineering regressions passed: strict OR threshold, final-only checkpoint, reject validation normalization, schedule indexing, historical failure peak retention, no test/pilot-weight read in train, unapproved-scope rejection. Existing model gradient/data acceptance remains unchanged and is not rerun. These are CPU checks, not evidence of actual S1 training. Source closure and inherited inputs are pinned before launch; actual started/worker progress alone establishes execution.
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Labelled S1 amendment; original pilot and scientific sources remain immutable."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked
|
||||
from phase2_h2_data import config as pilot_config
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
||||
old=pilot_config(Path(c['pilot_root']))
|
||||
if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed')
|
||||
if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap')
|
||||
return c,old
|
||||
|
||||
def gate(field_high,last_high):
|
||||
return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0))
|
||||
|
||||
def selected_checkpoint(completed_epochs):
|
||||
if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated')
|
||||
return 'last-epoch200.pt'
|
||||
|
||||
def stats_from_pilot(c,old):
|
||||
stats=read(checked(c['standardization_path'],c['standardization_sha256']))
|
||||
if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch')
|
||||
if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization')
|
||||
return stats
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Independent bounded S1 train guard; original pilot remains immutable."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h2_s1_common import config
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'s1.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
|
||||
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(root)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT')
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Start the one published H2 S1 training after host/GPU duplicate checks."""
|
||||
import hashlib,json,pathlib,subprocess,sys,time,psutil
|
||||
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17'
|
||||
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
|
||||
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
contract=json.loads((root/'contract.json').read_text())
|
||||
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
c=p.info['cmdline'] or []
|
||||
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2'
|
||||
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
|
||||
uuid=contract['gpu_uuid']
|
||||
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
|
||||
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
|
||||
unit='meltmind-phase2-h2-s1-20261117'
|
||||
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
|
||||
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)]
|
||||
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
|
||||
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
|
||||
print('SERVICE_LAUNCHED',unit,flush=True)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels."""
|
||||
import json,os,sys,time,traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import seed,peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root):
|
||||
c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stats=stats_from_pilot(c,old);save(out/'standardization.json',stats)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Engineering regressions for the independent S1 scope, no real data/GPU."""
|
||||
import ast,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import phase2_h2_s1_common as common
|
||||
from phase2_h2_model import lr_epoch
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
class S1Tests(unittest.TestCase):
|
||||
def test_gate_strict_and_or(self):
|
||||
self.assertFalse(common.gate(.005,0));self.assertTrue(common.gate(.00499,1));self.assertTrue(common.gate(.1,-1e-10));self.assertFalse(common.gate(None,float('nan')))
|
||||
def test_final_only(self):
|
||||
for e in [1,42,199,201]:
|
||||
with self.assertRaises(ValueError):common.selected_checkpoint(e)
|
||||
self.assertEqual(common.selected_checkpoint(200),'last-epoch200.pt')
|
||||
def test_normalization_rejects_validation(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d)/'s.json';p.write_text(json.dumps(dict(parts=['validation'],nodes=1,mean=[0]*19,scale=[1]*19)))
|
||||
c=dict(standardization_path=str(p),standardization_sha256='dummy')
|
||||
with patch.object(common,'checked',side_effect=lambda p,h:Path(p)):
|
||||
with self.assertRaises(ValueError):common.stats_from_pilot(c,dict(folds=[dict(learning=['learning'],learning_nodes=1)]))
|
||||
def test_schedule_index(self):
|
||||
self.assertAlmostEqual(lr_epoch(1),.0001);self.assertAlmostEqual(lr_epoch(10),.001);self.assertAlmostEqual(lr_epoch(200),.00001);self.assertLess(lr_epoch(11),lr_epoch(10))
|
||||
def test_history_peak_cannot_be_hidden(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);(p/'history').mkdir();(p/'history/001.json').write_text(json.dumps(dict(peak_reserved_bytes=16*2**30,peak_allocated_bytes=1)))
|
||||
(p/'result.json').write_text(json.dumps(dict(status='FAILED',peak_reserved_bytes=1)))
|
||||
tracker=PeakRecords();self.assertTrue(tracker.scan(p));(p/'history/001.json').unlink();self.assertTrue(tracker.scan(p))
|
||||
def test_train_does_not_open_test_or_pilot_weights(self):
|
||||
tree=ast.parse(Path('tools/phase2_h2_s1_worker.py').read_text())
|
||||
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call)]
|
||||
self.assertFalse(any(isinstance(n.func,ast.Attribute) and n.func.attr=='load' and isinstance(n.func.value,ast.Name) and n.func.value.id=='torch' for n in calls))
|
||||
roles=[n.args[3].value for n in calls if isinstance(n.func,ast.Name) and n.func.id=='load_part']
|
||||
self.assertEqual(sorted(roles),['learning','validation'])
|
||||
def test_unapproved_scope_rejected_before_sources(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);(p/'contract.json').write_text(json.dumps(dict(approval='wrong',mode='H2_S1_FOLD0_LAST200')))
|
||||
with self.assertRaises(PermissionError):common.config(p)
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user