From ea5ac05191fafda6829269c5fcdedac349ca6f3a Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 06:33:44 +0800 Subject: [PATCH] Freeze hash-bound zero-run v2 continuation with recorded idle settling --- .../continuation-contract.json | 79 ++++++++++++ .../synthetic-receipt.json | 6 + .../synthetic-tests.txt | 12 ++ ...s2-v2-idle-gate-continuation-2026-11-15.md | 11 ++ tools/run_s2_precision_continuation.py | 57 ++++++++ tools/s2_gpu_idle_gate.py | 53 ++++++++ tools/s2_precision_continuation_guard.py | 122 ++++++++++++++++++ tools/test_s2_gpu_idle_gate.py | 26 ++++ 8 files changed, 366 insertions(+) create mode 100644 artifacts/s2-pilot-v2-continuation-2026-11-15/continuation-contract.json create mode 100644 artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-receipt.json create mode 100644 artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-tests.txt create mode 100644 docs/s2-v2-idle-gate-continuation-2026-11-15.md create mode 100644 tools/run_s2_precision_continuation.py create mode 100644 tools/s2_gpu_idle_gate.py create mode 100644 tools/s2_precision_continuation_guard.py create mode 100644 tools/test_s2_gpu_idle_gate.py diff --git a/artifacts/s2-pilot-v2-continuation-2026-11-15/continuation-contract.json b/artifacts/s2-pilot-v2-continuation-2026-11-15/continuation-contract.json new file mode 100644 index 000000000..2d747c395 --- /dev/null +++ b/artifacts/s2-pilot-v2-continuation-2026-11-15/continuation-contract.json @@ -0,0 +1,79 @@ +{ + "version": "S2-v2-zero-run-engineering-continuation", + "authorizing_ruling": "1794695421.177269", + "source": "artifacts/s2-pilot-v2-2026-11-15", + "source_commit": "a15f2882ee2e8c3392c6b7c80aa521d7ba2ad58e", + "source_sha256": { + "artifacts/s2-pilot-v2-2026-11-15/pilot-plan.json": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae", + "artifacts/s2-pilot-v2-2026-11-15/training-contract.json": "0218b74f8d86ae4190e0b814f7c5c8a7224a616b94ad7587120156e755de2101", + "artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json": "96b95ec8dd0e37c38096dda9ec06257eb02662f8fd62cbe11b9ad58887d8376b", + "artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json": "45e9fe236d31dd7eabbbd46edce4fc7a80914c5a12088d1977de3633d79d6c2a", + "artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json": "337ea9552569db135e22b93f3e0b1557e03c5b56669cdb4d1bbcd391e6f67982", + "artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json": "e2794244d0664cc10aa3707a8da4f2d078946f985a1095f82a2f12b5310128a9" + }, + "code_sha256": { + "tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68", + "tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39", + "tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb", + "tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22", + "tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02", + "tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97", + "tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa", + "tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c", + "tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609", + "tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61", + "tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0", + "tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1", + "tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a", + "tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904", + "tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328", + "tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390", + "tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2", + "tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4", + "tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec", + "tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa", + "tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83", + "tools/run_s2_precision_continuation.py": "9fb5a22007788f0c18c77a82b6a8b76fbb53a98b49cc9251bc49a7a5b8312f2f", + "tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f" + }, + "prior_wall_seconds": 11.143491394002922, + "remaining_wall_seconds_at_entry": 7188.856508605997, + "total_v2_wall_limit_seconds": 7200, + "probe_reused": true, + "probe_rerun": false, + "E_max": 42, + "cosine_T_max": 42, + "training_runs_started_before_continuation": 0, + "idle_gate": { + "interval_seconds": 10, + "consecutive_idle_required": 3, + "selected_wait_limit_seconds": 600, + "memory_MiB_at_most": 128, + "utilization_percent": 0, + "compute_processes_required": 0, + "foreign_processes_never_killed": true, + "record_PID_user_memory_every_poll": true, + "switch_only_after_foreign_occupant_timeout_to_identical_V100": true + }, + "all_new_waits_and_setup_in_original_7200_seconds": true, + "original_busy_trigger_PID_user_memory": "UNKNOWN_NOT_RECORDED_AT_TRIGGER", + "original_post_stop_observation": { + "memory_MiB": 5, + "utilization_percent": 0, + "compute_processes": [] + }, + "scientific_code_changed": false, + "test_labels_read": false, + "no_automatic_retry": true +} diff --git a/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-receipt.json b/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-receipt.json new file mode 100644 index 000000000..8f70f65f7 --- /dev/null +++ b/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-receipt.json @@ -0,0 +1,6 @@ +{ + "status": "PASS", + "tests_run": 7, + "CPU_only": true, + "observed_unix": 1794695603.3577545 +} diff --git a/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-tests.txt b/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-tests.txt new file mode 100644 index 000000000..12991fbd4 --- /dev/null +++ b/artifacts/s2-pilot-v2-continuation-2026-11-15/synthetic-tests.txt @@ -0,0 +1,12 @@ +test_busy_then_three_consecutive_free (test_s2_gpu_idle_gate.IdleGateTests) ... ok +test_foreign_occupant_times_out_without_killing (test_s2_gpu_idle_gate.IdleGateTests) ... ok +test_short_remaining_budget_cannot_start_after_two_idle_polls (test_s2_gpu_idle_gate.IdleGateTests) ... ok +test_exact_device_memory_units (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok +test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok +test_missing_or_ambiguous_counters_fail_closed (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok +test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 0.036s + +OK diff --git a/docs/s2-v2-idle-gate-continuation-2026-11-15.md b/docs/s2-v2-idle-gate-continuation-2026-11-15.md new file mode 100644 index 000000000..906cf68cf --- /dev/null +++ b/docs/s2-v2-idle-gate-continuation-2026-11-15.md @@ -0,0 +1,11 @@ +# S2 v2 零训练run基础设施停止后的接续 + +Claude1794695421.177269接受v2 probe并裁量复用,不再probe。原session26792的PILOT_INCOMPLETE/runs=[]/SELECTED_GPU_BUSY_BEFORE_RUN及原11.143491394秒总账保持不变,固定提交a15f288。触发时PID、用户和占用值未由原门控保存,不能追溯确认占用者;停止后观测5MiB/0%/无compute进程不等于触发时证据。 + +新独立入口run_s2_precision_continuation.py绑定v2 plan、probe结果及guard、训练契约和两份终态SHA。根artifacts/s2-pilot-v2-continuation-2026-11-15。仅四训练run从头,不重新准备、probe或改变科学代码;E_max=cosine_T_max=42。原11.143491394秒已含probe,不重复扣probe;新入口剩余7188.856508606秒,全部新门控等待、设置和训练都计入同一7200秒。 + +入口对指定UUID每10秒采样,最多600秒,须连续3次memory≤128MiB、util0%、compute进程为空。每次保存PID/用户名/显存、设备值或采样错误;不杀其他进程。600秒后仍有外部占用才可查另一张名称完全相同的V100,并同样连续确认空闲,切换写入新的训练契约;否则停止。后续各run之间也使用同一门控,等待后重算剩余预算,不借新入口重置限额。 + +7项CPU合成检查覆盖busy→free、连续空闲被busy打断、外部占用600秒超时且不杀进程、余时不足不能通过3次检查,以及新guard的VRAM/主机内存/监控失败不进入第二run与可见性/计数器解析。原probe/precision源码pin不改。 + +实际启动和最终状态以新根started、idle-gates逐次JSONL、training-contract、guards/.started、run/started及complete/failure为准。pilot-started只是入口标记,不代表训练worker已启动。 diff --git a/tools/run_s2_precision_continuation.py b/tools/run_s2_precision_continuation.py new file mode 100644 index 000000000..d641dda9c --- /dev/null +++ b/tools/run_s2_precision_continuation.py @@ -0,0 +1,57 @@ +"""Engineering continuation of zero-run v2; reuse probe, preserve both stops.""" +import argparse +import fcntl +import json +from pathlib import Path +import time +import traceback +from freeze_g1_components import sha +from s2_e1_supervisor import save +from s2_gpu_idle_gate import inventory, recorded_wait +from s2_precision_continuation_guard import run as train_runs + + +def run(root): + with (root/'continuation.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + assert not (root/'started.json').exists(),'No automatic continuation retry' + contract=json.loads((root/'continuation-contract.json').read_text()) + for p,h in contract['code_sha256'].items():assert sha(p)==h,p + for p,h in contract['source_sha256'].items():assert sha(p)==h,p + source=Path(contract['source']);old=json.loads((source/'training-contract.json').read_text()) + ended=json.loads((source/'pilot-complete.json').read_text()) + assert ended['status']=='PILOT_INCOMPLETE' and ended['reason']=='SELECTED_GPU_BUSY_BEFORE_RUN' and ended['runs']==[] + previous=json.loads((source/'orchestrator-complete.json').read_text())['total_wall_seconds'] + assert previous==contract['prior_wall_seconds'] and old['E_max']==old['cosine_T_max']==42 + assert sha(old['timing_probe_receipt'])==old['timing_probe_sha256'] + started=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'continuation-contract.json'),prior_wall_seconds=previous)) + try: + names=inventory();selected=old['gpu_uuid'];assert 'V100' in names[selected] + gate=recorded_wait(root,'initial-selected',selected,min(600,7200-previous-(time.monotonic()-started)-5)) + switched=False + if gate['status']!='IDLE_CONFIRMED': + if not gate['last'].get('occupants'):raise RuntimeError('Selected GPU failed idle gate without identified foreign occupant; no switch') + for uuid,name in sorted(names.items()): + if uuid==selected or name!=names[selected]:continue + remaining=7200-previous-(time.monotonic()-started)-5 + if remaining<=20:break + alternative=recorded_wait(root,'alternative-'+uuid,uuid,min(600,remaining)) + if alternative['status']=='IDLE_CONFIRMED':selected=uuid;switched=True;break + if not switched:raise RuntimeError('No identical idle V100 after foreign-occupant timeout') + c=dict(old);c['gpu_uuid']=selected;c['code_sha256']=dict(old['code_sha256'],**contract['code_sha256']) + c['engineering_continuation']=dict(source=str(source),source_sha256=contract['source_sha256'], + prior_wall_seconds=previous,probe_reused=True,probe_rerun=False,gpu_switched=switched, + original_gpu_uuid=old['gpu_uuid'],selected_gpu_uuid=selected,ruling='1794695421.177269') + save(root/'training-contract.json',c) + train_runs(root,prior_wall_seconds=previous+time.monotonic()-started) + result=json.loads((root/'pilot-complete.json').read_text()) + save(root/'complete.json',dict(status=result['status'],selected_global_width=result['selected_global_width'], + cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous, + limit_seconds=7200,probe_reused=True,probe_rerun=False,test_labels_read=False)) + except Exception as exc: + save(root/'failure.json',dict(status='CONTINUATION_STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(), + cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,limit_seconds=7200));raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root) diff --git a/tools/s2_gpu_idle_gate.py b/tools/s2_gpu_idle_gate.py new file mode 100644 index 000000000..5e9422584 --- /dev/null +++ b/tools/s2_gpu_idle_gate.py @@ -0,0 +1,53 @@ +"""Bounded, recorded idle observations; never signal foreign processes.""" +import csv +import json +import subprocess +import time +from pathlib import Path +import psutil + + +def inventory(): + r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3) + return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())} + + +def snapshot(uuid): + r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) + rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1 + memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100 + r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) + occupants=[] + for row in csv.reader(r.stdout.splitlines()): + if not row or row[0].strip()!=uuid:continue + pid=int(row[1]);user=None + try:user=psutil.Process(pid).username() + except (psutil.NoSuchProcess,psutil.AccessDenied):pass + occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip())) + return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants, + idle=memory<=128 and util==0 and not occupants) + + +def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep): + begin=clock();consecutive=0;count=0;last=None + while True: + try:last=sample(uuid) + except Exception as exc: + last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[]) + count+=1;consecutive=consecutive+1 if last['idle'] else 0 + record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin)) + if clock()-begin>limit_seconds:break + if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) + remaining=limit_seconds-(clock()-begin) + if remaining<=0:break + sleep(min(10,remaining)) + return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) + + +def recorded_wait(root,name,uuid,limit_seconds): + path=Path(root)/'idle-gates';path.mkdir(exist_ok=True) + def record(row): + with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n') + result=wait_idle(uuid,limit_seconds,record) + with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n') + return result diff --git a/tools/s2_precision_continuation_guard.py b/tools/s2_precision_continuation_guard.py new file mode 100644 index 000000000..80961aef1 --- /dev/null +++ b/tools/s2_precision_continuation_guard.py @@ -0,0 +1,122 @@ +"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import psutil +from freeze_g1_components import sha +from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage +from s2_pilot_train import select_capacity +from s2_gpu_idle_gate import recorded_wait + + +def parse_device_usage(text): + lines=text.strip().splitlines() + if len(lines)!=1:raise ValueError('Expected exactly one GPU') + fields=lines[0].split(',') + if len(fields)!=2:raise ValueError('Malformed GPU usage row') + memory,util=map(lambda s:int(s.strip()),fields) + if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters') + return memory*2**20,util + + +def device_usage(uuid): + r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2) + return parse_device_usage(r.stdout) + + +def active_training_pids(parent,root,index): + ids=[] + try:children=psutil.Process(parent).children(recursive=True) + except psutil.NoSuchProcess:return ids + for p in children: + try: + cmd=p.cmdline() + if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid) + except psutil.NoSuchProcess:pass + return ids + + +def guarded_worker(root,spec,i,uuid,wall_limit,command): + run_start=time.monotonic();initial,util=device_usage(uuid) + if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN') + done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0) + def watch(): + previous=time.monotonic() + while not done.is_set(): + now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now + try: + used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) + if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT' + _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) + if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT' + if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT' + except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc) + if monitor['reason']: + for pid in active_training_pids(os.getpid(),root,i):stop_group(pid) + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + wall=wall_limit-(time.monotonic()-run_start)-2.5 + if wall<=0:raise RuntimeError('No budget left before spawn') + r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall, + tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid) + finally: + done.set();thread.join(timeout=3) + total_wall=time.monotonic()-run_start + result_path=root/spec['name']/'result.json' + result=json.loads(result_path.read_text()) if result_path.exists() else None + if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason']) + if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT') + if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT') + r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall, + result_sha256=sha(result_path) if result else None) + save(root/(spec['name']+'-guard-receipt.json'),r) + return r,result + + +def run(root,prior_wall_seconds=0.): + with (root/'pilot.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return run_locked(root,prior_wall_seconds) + + +def run_locked(root,prior_wall_seconds=0.): + c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid'] + prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.)) + assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)] + for p,h in c['code_sha256'].items():assert sha(p)==h,p + assert not (root/'pilot-started.json').exists(),'No automatic pilot retry' + start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) + rows=[];scores={};reason=None + for i,spec in enumerate(c['runs']): + remaining=7200-prior_wall_seconds-(time.monotonic()-start) + if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break + # Initial readiness was proved by the continuation entry; subsequent + # contexts also need a settling window. All waits consume the same budget. + if i>0: + gate=recorded_wait(root,spec['name'],uuid,min(600,remaining-3)) + if gate['status']!='IDLE_CONFIRMED':reason='GPU_IDLE_TIMEOUT_BEFORE_RUN';break + remaining=7200-prior_wall_seconds-(time.monotonic()-start) + if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break + command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)] + try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command) + except Exception as exc:reason=str(exc);break + rows.append(dict(run=spec,**r)) + if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break + scores[(spec['arm'],spec['width'])]=result['best_validation_score'] + if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT' + selected=select_capacity(scores) if len(scores)==4 and reason is None else None + summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected, + selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required', + pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0) + save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root) diff --git a/tools/test_s2_gpu_idle_gate.py b/tools/test_s2_gpu_idle_gate.py new file mode 100644 index 000000000..2917adf63 --- /dev/null +++ b/tools/test_s2_gpu_idle_gate.py @@ -0,0 +1,26 @@ +import unittest +from s2_gpu_idle_gate import wait_idle + + +class IdleGateTests(unittest.TestCase): + def exercise(self,states,limit): + now=[0.];events=[];iterator=iter(states);last=[states[-1]] + def sample(uuid): + value=next(iterator,last[0]);last[0]=value + return dict(uuid=uuid,idle=value,occupants=[] if value else [dict(pid=123,user='foreign',memory_MiB='1000')]) + result=wait_idle('synthetic',limit,events.append,sample=sample,clock=lambda:now[0],sleep=lambda x:now.__setitem__(0,now[0]+x)) + return result,events + def test_busy_then_three_consecutive_free(self): + result,events=self.exercise([False,True,False,True,True,True],600) + self.assertEqual(result['status'],'IDLE_CONFIRMED');self.assertEqual(result['wall_seconds'],50) + self.assertEqual([x['consecutive_idle'] for x in events],[0,1,0,1,2,3]) + def test_foreign_occupant_times_out_without_killing(self): + result,events=self.exercise([False],600) + self.assertEqual(result['status'],'IDLE_TIMEOUT');self.assertEqual(result['wall_seconds'],600) + self.assertEqual(result['last']['occupants'][0]['pid'],123);self.assertEqual(len(events),61) + def test_short_remaining_budget_cannot_start_after_two_idle_polls(self): + result,_=self.exercise([True],10) + self.assertEqual(result['status'],'IDLE_TIMEOUT') + + +if __name__=='__main__':unittest.main(verbosity=2)