Fix Track A recovery projection isolation and bounded prestart deadline
This commit is contained in:
1 parent
ef46d35772
commit
0886d459e6
65 files changed
+25625
-46
No files matched your search
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"timestamp": "2026-11-18T11:36:22.777820+00:00",
|
||||
"binding_sha256": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"independent_tests": {
|
||||
"count": 21,
|
||||
"status": "PASS",
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"wall_seconds": 0.337,
|
||||
"session": 19562
|
||||
},
|
||||
"hash_validation": {
|
||||
"code": 27,
|
||||
"sources": 60,
|
||||
"local_implementation_files": 40,
|
||||
"original_infrastructure_stop_files": 952,
|
||||
"original_stop_commit": "a0a32fbf2d438cc6d39715cb2742f74ee415ebfe",
|
||||
"seed1_code_pins": 51,
|
||||
"status": "PASS"
|
||||
},
|
||||
"config": "PASS",
|
||||
"reuse_identity": "PASS; old0/1/2 guard/worker/final200/steps/checkpoint and normalization hashes verified without model deserialization",
|
||||
"ledger": {
|
||||
"old_once": 47943.94719455298,
|
||||
"new_train_cap": 106856.05280544702,
|
||||
"postfit": 7200,
|
||||
"cumulative_cap": 162000,
|
||||
"per_recovery_fold": 30960,
|
||||
"initial_cumulative_projection": 117063.94719455298
|
||||
},
|
||||
"reproducer": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py",
|
||||
"sha256": "a01d403f400b4adabfca32c3291dfd6228878634b963353bccd274a4924da696",
|
||||
"method": "Exact queue cost-loop and guard idle-loop AST fragments with synthetic files/samples/clock only",
|
||||
"own_projection": {
|
||||
"fold": 3,
|
||||
"epoch_seconds": 1000,
|
||||
"charged": 160,
|
||||
"rejected_projection": 249030,
|
||||
"actual_global_projection": 335133.947194553,
|
||||
"actual_plus_survivor_full_reserve_and_postfit": 86263.94719455298,
|
||||
"own_stop": "OWN_PROJECTED_CAP",
|
||||
"also_global_stop": "CUMULATIVE_PROJECTED_CAP"
|
||||
},
|
||||
"idle_loop": {
|
||||
"expired_elapsed": 30961,
|
||||
"remaining_cap": 30960,
|
||||
"continued_queries": 14,
|
||||
"elapsed_checks": 0,
|
||||
"maximum_error_streak": 2,
|
||||
"monitor_failed": false,
|
||||
"ended_by_synthetic_busy_sample": true
|
||||
}
|
||||
},
|
||||
"postfit_compatibility": "Old983bba99 remains incompatible; complete success requires new mixed-source identity-only postfit binding, no partial-cohort scoring accepted here",
|
||||
"root_markers": {
|
||||
"review.json": false,
|
||||
"started.json": false,
|
||||
"complete.json": false
|
||||
},
|
||||
"old_failures_preserved": true,
|
||||
"remote_publication_independently_verified": false,
|
||||
"real_gpu_queries": false,
|
||||
"scientific_workloads": false,
|
||||
"target_values_loaded": false,
|
||||
"model_values_loaded": false,
|
||||
"source_edits": false,
|
||||
"git_writes": false
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Exact recovery AST fragments with synthetic files, clock and GPU samples only."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from track_a_recovery_common import project_cell,project_total,OLD
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
def fragment(path,predicate):
|
||||
tree=ast.parse(Path(path).read_text())
|
||||
node=next(n for n in ast.walk(tree) if predicate(n))
|
||||
return compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),path,'exec')
|
||||
|
||||
outputs={}
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);save(root/'contract.json',{})
|
||||
for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True)
|
||||
save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000))
|
||||
own=[];glob=[]
|
||||
c=dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200)
|
||||
env=dict(root=root,c=c,active={3:object(),4:object()},terminal={},estimates={},last_epoch={},
|
||||
read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,
|
||||
live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,
|
||||
time=SimpleNamespace(time=lambda:0),own_stop=lambda i,why:own.append((i,why)),
|
||||
global_stop=lambda why:glob.append(why))
|
||||
code=fragment('tools/track_a_recovery_queue.py',lambda n:isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)')
|
||||
exec(code,env)
|
||||
assert own==[(3,'OWN_PROJECTED_CAP')] and glob==['CUMULATIVE_PROJECTED_CAP']
|
||||
total=env['total']['cumulative_projected_gpu_seconds']
|
||||
actual_plus_survivor_reserve=OLD+160+30960+7200
|
||||
assert actual_plus_survivor_reserve<162000
|
||||
outputs['own_projection_escalates_to_global']=dict(own_stops=own,global_stops=glob,
|
||||
rejected_fold_projection=env['cell']['projected_gpu_seconds'],global_projection=total,
|
||||
actual_failed_fold_plus_survivor_full_reserve_and_postfit=actual_plus_survivor_reserve,
|
||||
explanation='Known locally rejected future work still enters total and globally stops the other fold')
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);calls=[];elapsed_calls=[];monitor=Monitor(root/'errors.jsonl')
|
||||
def query(uuid):
|
||||
calls.append(uuid)
|
||||
if len(calls)==14:return 1024**3,50 # End replay using existing busy exit.
|
||||
if len(calls)%3:raise TimeoutError('synthetic intermittent GPU query failure')
|
||||
return 0,0
|
||||
def elapsed():
|
||||
elapsed_calls.append(True)
|
||||
return 30961.0 # Already exceeds remaining per-fold allowance.
|
||||
env=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],
|
||||
device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=30960),
|
||||
elapsed=elapsed,time=SimpleNamespace(time=lambda:0,sleep=lambda s:None),save=save)
|
||||
code=fragment('tools/track_a_recovery_guard.py',lambda n:isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3')
|
||||
try:exec(code,env)
|
||||
except RuntimeError as exc:reason=str(exc)
|
||||
else:raise AssertionError('Expected synthetic busy exit')
|
||||
assert reason=='SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED'
|
||||
assert len(calls)==14 and elapsed_calls==[] and not monitor.failed
|
||||
outputs['prestart_idle_loop_has_no_deadline']=dict(already_expired_elapsed=30961,
|
||||
remaining_cap=30960,query_calls_after_expiry=len(calls),elapsed_checks=len(elapsed_calls),
|
||||
max_consecutive_errors=monitor.max_streaks,monitor_failed=monitor.failed,
|
||||
synthetic_termination=reason,explanation='Repeating two failures then one valid idle sample resets monitor streak but never gets three idle successes; no elapsed check bounds this loop')
|
||||
|
||||
print(json.dumps(dict(reproductions=outputs,real_gpu_queries=False,science_executed=False,
|
||||
source_hashes={str(p):sha(p) for p in map(Path,['tools/track_a_recovery_queue.py','tools/track_a_recovery_guard.py'])}),indent=2))
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"reviewed_commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"implementation_proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"comparison_contract_sha256": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5"
|
||||
},
|
||||
"verdict": "BLOCKED",
|
||||
"reviewer": "MeltAssist",
|
||||
"timestamp": "2026-11-18T11:36:22.777820+00:00",
|
||||
"review_scope": "Same unique Track A infrastructure recovery narrow delta requested1795001544; directly read Claude1795000465/0626",
|
||||
"evidence_path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json",
|
||||
"evidence_sha256": "a6c9eb98ca7d3f5e95e0b548aafcea3d980477ffba11ebb19c4809bc7d1ba386",
|
||||
"findings": [
|
||||
{
|
||||
"id": "RECOVERY-OWN-PROJECTION-ISOLATION",
|
||||
"priority": "P1",
|
||||
"file": "tools/track_a_recovery_queue.py",
|
||||
"location": "active epoch cost-projection block",
|
||||
"title": "Do not promote an already rejected own-fold projection into a global stop",
|
||||
"observed": "The loop installs an over-cap cell estimate, computes total with its full rejected future work, sets own stop and then global stop when that inflated total exceeds162000. Synthetic actual block: job3 projected249030 against30960, causing both OWN_PROJECTED_CAP and CUMULATIVE_PROJECTED_CAP; failed-job actual160 + survivor full30960 + post7200 + old47943.9472 is only86263.9472. The other fold is then terminated despite its continuation fitting budget.",
|
||||
"requested_fix": "Separate locally stopping job from continuing future work before deciding global projected exhaustion. Keep its raw rejected projection and all actual/termination costs, reserve bounded cleanup or the already approved remaining own cap until terminal, and keep checking real cumulative budget/shared integrity globally. Test an extreme own projection with an affordable healthy survivor; no larger budget or retry."
|
||||
},
|
||||
{
|
||||
"id": "RECOVERY-PRESTART-DEADLINE",
|
||||
"priority": "P1",
|
||||
"file": "tools/track_a_recovery_guard.py",
|
||||
"location": "while success < 3, before local/gpu watcher start",
|
||||
"title": "Bound the pre-worker idle sampling loop by its remaining per-fold time",
|
||||
"observed": "Idle loop has no elapsed/window check; local time watcher only starts after three successful idle samples. Repeated two query failures then one success resets monitor streak below3 and also prevents three consecutive successes, so the loop can continue beyond remaining30960. Exact-loop replay with elapsed30961 performed14 queries and zero elapsed checks, until synthetic busy ended it. The queue active path has no per-fold actual deadline before epoch receipts to compensate.",
|
||||
"requested_fix": "Check the remaining job deadline before/after idle sampling and before sleeping, retain bounded10s query/cleanup margin and fail only this fold when its remaining window expires. Preserve transient-error logs and per-operation3-failure policy. Add alternating recoverable-error/idle-success fake-clock test that expires without a third error and without launching a worker."
|
||||
}
|
||||
],
|
||||
"passed_checks": [
|
||||
"21 independent CPU synthetic tests PASS; worker scientific AST unchanged apart from config/first-gate import and3/4 execution boundary",
|
||||
"Original0/1/2 PASS checkpoints and learning normalization hash reuse verified with952-file immutable stop proof; no reused fold can execute",
|
||||
"Old47943.94719455298 charged once, new train106856.05280544702, post7200, cumulative162000; terminal failed costs retained, initial reserve passes",
|
||||
"10s GPU query and per-operation three-consecutive-failure monitor, success reset, missing-not-breach, retained errors and local-fast/GPU split reviewed; synthetic real child resource breach immediate and transient tree failures tested",
|
||||
"Ordinary collect failure and dispatch monitor failure isolate the job; missing recovery model prevents canonical five-fold adoption",
|
||||
"Seed1 pins untouched; GPU2/3 idle checks and no further recovery retained; oldpostfit not incorrectly reused"
|
||||
],
|
||||
"limitations": [
|
||||
"Two blockers are synthetic/code-review findings, not actual recovery failures; no recovery execution occurred",
|
||||
"Current postfit compatibility is not approved by this review; full success only emits rebind-required and missing folds cannot bypass completeness",
|
||||
"Original stopped-coordinator CPU accounting gaps and historical failures remain; old guard CPU includes worker and must not be added again",
|
||||
"No scientific array/target/model values loaded, no real GPU query, no source or Git mutation; only own review artifacts written",
|
||||
"Local immutable Git proofs verified; no remote publication raw-byte claim"
|
||||
]
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"timestamp": "2026-11-18T11:36:22.777820+00:00",
|
||||
"binding_sha256": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"independent_tests": {
|
||||
"count": 21,
|
||||
"status": "PASS",
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"wall_seconds": 0.337,
|
||||
"session": 19562
|
||||
},
|
||||
"hash_validation": {
|
||||
"code": 27,
|
||||
"sources": 60,
|
||||
"local_implementation_files": 40,
|
||||
"original_infrastructure_stop_files": 952,
|
||||
"original_stop_commit": "a0a32fbf2d438cc6d39715cb2742f74ee415ebfe",
|
||||
"seed1_code_pins": 51,
|
||||
"status": "PASS"
|
||||
},
|
||||
"config": "PASS",
|
||||
"reuse_identity": "PASS; old0/1/2 guard/worker/final200/steps/checkpoint and normalization hashes verified without model deserialization",
|
||||
"ledger": {
|
||||
"old_once": 47943.94719455298,
|
||||
"new_train_cap": 106856.05280544702,
|
||||
"postfit": 7200,
|
||||
"cumulative_cap": 162000,
|
||||
"per_recovery_fold": 30960,
|
||||
"initial_cumulative_projection": 117063.94719455298
|
||||
},
|
||||
"reproducer": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py",
|
||||
"sha256": "a01d403f400b4adabfca32c3291dfd6228878634b963353bccd274a4924da696",
|
||||
"method": "Exact queue cost-loop and guard idle-loop AST fragments with synthetic files/samples/clock only",
|
||||
"own_projection": {
|
||||
"fold": 3,
|
||||
"epoch_seconds": 1000,
|
||||
"charged": 160,
|
||||
"rejected_projection": 249030,
|
||||
"actual_global_projection": 335133.947194553,
|
||||
"actual_plus_survivor_full_reserve_and_postfit": 86263.94719455298,
|
||||
"own_stop": "OWN_PROJECTED_CAP",
|
||||
"also_global_stop": "CUMULATIVE_PROJECTED_CAP"
|
||||
},
|
||||
"idle_loop": {
|
||||
"expired_elapsed": 30961,
|
||||
"remaining_cap": 30960,
|
||||
"continued_queries": 14,
|
||||
"elapsed_checks": 0,
|
||||
"maximum_error_streak": 2,
|
||||
"monitor_failed": false,
|
||||
"ended_by_synthetic_busy_sample": true
|
||||
}
|
||||
},
|
||||
"postfit_compatibility": "Old983bba99 remains incompatible; complete success requires new mixed-source identity-only postfit binding, no partial-cohort scoring accepted here",
|
||||
"root_markers": {
|
||||
"review.json": false,
|
||||
"started.json": false,
|
||||
"complete.json": false
|
||||
},
|
||||
"old_failures_preserved": true,
|
||||
"remote_publication_independently_verified": false,
|
||||
"real_gpu_queries": false,
|
||||
"scientific_workloads": false,
|
||||
"target_values_loaded": false,
|
||||
"model_values_loaded": false,
|
||||
"source_edits": false,
|
||||
"git_writes": false
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Exact recovery AST fragments with synthetic files, clock and GPU samples only."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from track_a_recovery_common import project_cell,project_total,OLD
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
def fragment(path,predicate):
|
||||
tree=ast.parse(Path(path).read_text())
|
||||
node=next(n for n in ast.walk(tree) if predicate(n))
|
||||
return compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),path,'exec')
|
||||
|
||||
outputs={}
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);save(root/'contract.json',{})
|
||||
for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True)
|
||||
save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000))
|
||||
own=[];glob=[]
|
||||
c=dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200)
|
||||
env=dict(root=root,c=c,active={3:object(),4:object()},terminal={},estimates={},last_epoch={},
|
||||
read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,
|
||||
live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,
|
||||
time=SimpleNamespace(time=lambda:0),own_stop=lambda i,why:own.append((i,why)),
|
||||
global_stop=lambda why:glob.append(why))
|
||||
code=fragment('tools/track_a_recovery_queue.py',lambda n:isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)')
|
||||
exec(code,env)
|
||||
assert own==[(3,'OWN_PROJECTED_CAP')] and glob==['CUMULATIVE_PROJECTED_CAP']
|
||||
total=env['total']['cumulative_projected_gpu_seconds']
|
||||
actual_plus_survivor_reserve=OLD+160+30960+7200
|
||||
assert actual_plus_survivor_reserve<162000
|
||||
outputs['own_projection_escalates_to_global']=dict(own_stops=own,global_stops=glob,
|
||||
rejected_fold_projection=env['cell']['projected_gpu_seconds'],global_projection=total,
|
||||
actual_failed_fold_plus_survivor_full_reserve_and_postfit=actual_plus_survivor_reserve,
|
||||
explanation='Known locally rejected future work still enters total and globally stops the other fold')
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);calls=[];elapsed_calls=[];monitor=Monitor(root/'errors.jsonl')
|
||||
def query(uuid):
|
||||
calls.append(uuid)
|
||||
if len(calls)==14:return 1024**3,50 # End replay using existing busy exit.
|
||||
if len(calls)%3:raise TimeoutError('synthetic intermittent GPU query failure')
|
||||
return 0,0
|
||||
def elapsed():
|
||||
elapsed_calls.append(True)
|
||||
return 30961.0 # Already exceeds remaining per-fold allowance.
|
||||
env=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],
|
||||
device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=30960),
|
||||
elapsed=elapsed,time=SimpleNamespace(time=lambda:0,sleep=lambda s:None),save=save)
|
||||
code=fragment('tools/track_a_recovery_guard.py',lambda n:isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3')
|
||||
try:exec(code,env)
|
||||
except RuntimeError as exc:reason=str(exc)
|
||||
else:raise AssertionError('Expected synthetic busy exit')
|
||||
assert reason=='SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED'
|
||||
assert len(calls)==14 and elapsed_calls==[] and not monitor.failed
|
||||
outputs['prestart_idle_loop_has_no_deadline']=dict(already_expired_elapsed=30961,
|
||||
remaining_cap=30960,query_calls_after_expiry=len(calls),elapsed_checks=len(elapsed_calls),
|
||||
max_consecutive_errors=monitor.max_streaks,monitor_failed=monitor.failed,
|
||||
synthetic_termination=reason,explanation='Repeating two failures then one valid idle sample resets monitor streak but never gets three idle successes; no elapsed check bounds this loop')
|
||||
|
||||
print(json.dumps(dict(reproductions=outputs,real_gpu_queries=False,science_executed=False,
|
||||
source_hashes={str(p):sha(p) for p in map(Path,['tools/track_a_recovery_queue.py','tools/track_a_recovery_guard.py'])}),indent=2))
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"reviewed_commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"implementation_proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"comparison_contract_sha256": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5"
|
||||
},
|
||||
"verdict": "BLOCKED",
|
||||
"reviewer": "MeltAssist",
|
||||
"timestamp": "2026-11-18T11:36:22.777820+00:00",
|
||||
"review_scope": "Same unique Track A infrastructure recovery narrow delta requested1795001544; directly read Claude1795000465/0626",
|
||||
"evidence_path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json",
|
||||
"evidence_sha256": "a6c9eb98ca7d3f5e95e0b548aafcea3d980477ffba11ebb19c4809bc7d1ba386",
|
||||
"findings": [
|
||||
{
|
||||
"id": "RECOVERY-OWN-PROJECTION-ISOLATION",
|
||||
"priority": "P1",
|
||||
"file": "tools/track_a_recovery_queue.py",
|
||||
"location": "active epoch cost-projection block",
|
||||
"title": "Do not promote an already rejected own-fold projection into a global stop",
|
||||
"observed": "The loop installs an over-cap cell estimate, computes total with its full rejected future work, sets own stop and then global stop when that inflated total exceeds162000. Synthetic actual block: job3 projected249030 against30960, causing both OWN_PROJECTED_CAP and CUMULATIVE_PROJECTED_CAP; failed-job actual160 + survivor full30960 + post7200 + old47943.9472 is only86263.9472. The other fold is then terminated despite its continuation fitting budget.",
|
||||
"requested_fix": "Separate locally stopping job from continuing future work before deciding global projected exhaustion. Keep its raw rejected projection and all actual/termination costs, reserve bounded cleanup or the already approved remaining own cap until terminal, and keep checking real cumulative budget/shared integrity globally. Test an extreme own projection with an affordable healthy survivor; no larger budget or retry."
|
||||
},
|
||||
{
|
||||
"id": "RECOVERY-PRESTART-DEADLINE",
|
||||
"priority": "P1",
|
||||
"file": "tools/track_a_recovery_guard.py",
|
||||
"location": "while success < 3, before local/gpu watcher start",
|
||||
"title": "Bound the pre-worker idle sampling loop by its remaining per-fold time",
|
||||
"observed": "Idle loop has no elapsed/window check; local time watcher only starts after three successful idle samples. Repeated two query failures then one success resets monitor streak below3 and also prevents three consecutive successes, so the loop can continue beyond remaining30960. Exact-loop replay with elapsed30961 performed14 queries and zero elapsed checks, until synthetic busy ended it. The queue active path has no per-fold actual deadline before epoch receipts to compensate.",
|
||||
"requested_fix": "Check the remaining job deadline before/after idle sampling and before sleeping, retain bounded10s query/cleanup margin and fail only this fold when its remaining window expires. Preserve transient-error logs and per-operation3-failure policy. Add alternating recoverable-error/idle-success fake-clock test that expires without a third error and without launching a worker."
|
||||
}
|
||||
],
|
||||
"passed_checks": [
|
||||
"21 independent CPU synthetic tests PASS; worker scientific AST unchanged apart from config/first-gate import and3/4 execution boundary",
|
||||
"Original0/1/2 PASS checkpoints and learning normalization hash reuse verified with952-file immutable stop proof; no reused fold can execute",
|
||||
"Old47943.94719455298 charged once, new train106856.05280544702, post7200, cumulative162000; terminal failed costs retained, initial reserve passes",
|
||||
"10s GPU query and per-operation three-consecutive-failure monitor, success reset, missing-not-breach, retained errors and local-fast/GPU split reviewed; synthetic real child resource breach immediate and transient tree failures tested",
|
||||
"Ordinary collect failure and dispatch monitor failure isolate the job; missing recovery model prevents canonical five-fold adoption",
|
||||
"Seed1 pins untouched; GPU2/3 idle checks and no further recovery retained; oldpostfit not incorrectly reused"
|
||||
],
|
||||
"limitations": [
|
||||
"Two blockers are synthetic/code-review findings, not actual recovery failures; no recovery execution occurred",
|
||||
"Current postfit compatibility is not approved by this review; full success only emits rebind-required and missing folds cannot bypass completeness",
|
||||
"Original stopped-coordinator CPU accounting gaps and historical failures remain; old guard CPU includes worker and must not be added again",
|
||||
"No scientific array/target/model values loaded, no real GPU query, no source or Git mutation; only own review artifacts written",
|
||||
"Local immutable Git proofs verified; no remote publication raw-byte claim"
|
||||
]
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
test_actual_charge_floor_and_nonfinite_rejection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_actual_collect_failure_does_not_stop_other_job (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_device_query_uses_ten_seconds_and_parses (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_failed_terminal_cost_not_dropped_and_replaces_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_fast_local_guard_not_blocked_on_gpu_query (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_final_sample_retry_bounded_at_three (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_missing_is_not_zero_resource_sample (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_no_partial_model_adoption (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_old_once_and_only_two_new_jobs_in_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_original_worker_science_ast_unchanged (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_own_cost_gate_stop_does_not_block_other_fold (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_reused_fold_execution_rejected (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_success_in_other_stream_does_not_hide_gpu_failures (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_real_breach_fails_without_three_monitor_errors (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_three_failed_tree_samples_stop_only_child (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_transient_tree_failures_recover_and_log (tools.test_track_a_recovery.RecoveryTests) ... FAIL
|
||||
test_third_consecutive_failure_is_terminal (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_transient_logged_monitor_errors_can_pass_but_fatal_cannot (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_two_failures_then_success_reset_and_retain_evidence (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
|
||||
======================================================================
|
||||
FAIL: test_supervisor_transient_tree_failures_recover_and_log (tools.test_track_a_recovery.RecoveryTests)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py", line 132, in test_supervisor_transient_tree_failures_recover_and_log
|
||||
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],2);self.assertFalse(r['monitor_policy']['failed'])
|
||||
AssertionError: 'STOPPED_OR_FAILED' != 'PASS'
|
||||
- STOPPED_OR_FAILED
|
||||
+ PASS
|
||||
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 19 tests in 0.231s
|
||||
|
||||
FAILED (failures=1)
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
test_actual_charge_floor_and_nonfinite_rejection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_actual_collect_failure_does_not_stop_other_job (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_device_query_uses_ten_seconds_and_parses (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_dispatch_busy_gpu_never_launches_or_changes_other_queue (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_dispatch_three_query_failures_isolate_unlaunched_job (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_failed_terminal_cost_not_dropped_and_replaces_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_fast_local_guard_not_blocked_on_gpu_query (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_final_sample_retry_bounded_at_three (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_missing_is_not_zero_resource_sample (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_no_partial_model_adoption (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_old_once_and_only_two_new_jobs_in_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_original_worker_science_ast_unchanged (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_own_cost_gate_stop_does_not_block_other_fold (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_reused_fold_execution_rejected (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_success_in_other_stream_does_not_hide_gpu_failures (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_real_breach_fails_without_three_monitor_errors (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_three_failed_tree_samples_stop_only_child (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_transient_tree_failures_recover_and_log (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_third_consecutive_failure_is_terminal (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_transient_logged_monitor_errors_can_pass_but_fatal_cannot (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_two_failures_then_success_reset_and_retain_evidence (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 21 tests in 0.319s
|
||||
|
||||
OK
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"unix": 1795001369.1961312,
|
||||
"gpu_observations": [
|
||||
{
|
||||
"gpu_index": 2,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"query_start": 1795001369.060091,
|
||||
"query_end": 1795001369.105409,
|
||||
"memory_bytes": 9437184,
|
||||
"utilization": 0,
|
||||
"idle": true,
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"gpu_index": 3,
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"query_start": 1795001369.1054165,
|
||||
"query_end": 1795001369.1506112,
|
||||
"memory_bytes": 1928331264,
|
||||
"utilization": 14,
|
||||
"idle": false,
|
||||
"error": null
|
||||
}
|
||||
],
|
||||
"seed1_live": {
|
||||
"stage": "SEED1_TRAINING",
|
||||
"unix": 1795001368.8062162,
|
||||
"active": [
|
||||
{
|
||||
"id": 0,
|
||||
"guard_pid": 1476627
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"guard_pid": 1476628
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"guard_pid": 1598086
|
||||
}
|
||||
],
|
||||
"terminal": [],
|
||||
"pending": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"charged_training_gpu_seconds": 20089.052661241964,
|
||||
"postfit_gpu_reserve": 7200
|
||||
},
|
||||
"seed1_processes": [
|
||||
{
|
||||
"job": 0,
|
||||
"guard_pid": 1476627,
|
||||
"children": [
|
||||
1476707
|
||||
]
|
||||
},
|
||||
{
|
||||
"job": 1,
|
||||
"guard_pid": 1476628,
|
||||
"children": [
|
||||
1476726
|
||||
]
|
||||
},
|
||||
{
|
||||
"job": 2,
|
||||
"guard_pid": 1598086,
|
||||
"children": [
|
||||
1598144
|
||||
]
|
||||
}
|
||||
],
|
||||
"seed1_pins": "PASS",
|
||||
"original_track_a_pins": "PASS",
|
||||
"recovery_root_exists": false,
|
||||
"no_gpu_workload": true,
|
||||
"no_launch": true,
|
||||
"launch_requires_new_actual_idle_check": true
|
||||
}
|
||||
Whitespace-only changes.
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 0.023683,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "synthetic",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"cpu_seconds": 0.023683
|
||||
}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
+2
@@ -0,0 +1,2 @@
|
||||
{"unix": 1795001297.4464219, "operation": "tree_usage", "type": "OSError", "message": "synthetic transient", "consecutive_failures": 1, "failed": false}
|
||||
{"unix": 1795001297.4568598, "operation": "tree_usage", "type": "OSError", "message": "synthetic transient", "consecutive_failures": 2, "failed": false}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"-c",
|
||||
"import time; time.sleep(.15)"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": "rss_limit_after_exit",
|
||||
"cpu_seconds": 0.023683,
|
||||
"wall_seconds": 0.1828980860300362,
|
||||
"peak_tree_rss_bytes": 404328448,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 8192,
|
||||
"max_poll_gap_seconds": 0.011175077059306204,
|
||||
"poll_target_seconds": 0.01,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"gpu_uuid": null,
|
||||
"retry": false,
|
||||
"monitor_errors": 2,
|
||||
"last_monitor_error": {
|
||||
"unix": 1795001297.4568598,
|
||||
"operation": "tree_usage",
|
||||
"type": "OSError",
|
||||
"message": "synthetic transient",
|
||||
"consecutive_failures": 2,
|
||||
"failed": false
|
||||
},
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 2,
|
||||
"streaks": {
|
||||
"initial_disk": 0,
|
||||
"tree_usage": 0,
|
||||
"live_disk": 0,
|
||||
"final_disk": 0
|
||||
},
|
||||
"max_streaks": {
|
||||
"tree_usage": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"-c",
|
||||
"import time; time.sleep(.15)"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 10,
|
||||
"job_cpu_seconds": 10,
|
||||
"job_wall_seconds": 5,
|
||||
"tree_rss_bytes": 100000000,
|
||||
"disk_bytes": 1000000,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1795001297.444848
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# Track A 唯一标记基础设施恢复:待窄差异审阅
|
||||
|
||||
已直接读取Claude1795000465.748129与1795000626.905989:原科学失败不重试规则保持;此次外层监控基础设施abort获准唯一标记recovery。当前只准备独立入口与审阅绑定,不训练、不建立自动waiter、不启动postfit。21项CPU合成检查PASS,目标是交MeltAssist窄delta审阅复用、监控、失败隔离和账本。
|
||||
|
||||
独立root:`/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18`。原root及STOP、三PASS、两中断、监控错误、原CPU记录缺口全部不改;原证据冻结a0a32fbf/共享32bd61e1仍有效。精确新五字段以root内`expected-review-binding.json`为准,实际新PASS前不得运行。
|
||||
|
||||
## 复用与科学身份
|
||||
|
||||
fold0/1/2仅哈希复用原last-epoch200检查点与learning-only19列标准化;逐项验证原guard/worker结果、200epochs/steps、角色与不可变STOP冻结。旧模型不加载、不重训。fold3/4从头运行同seed0、原615 v4输入/五split成员/原时钟/200epoch完整日程/原优化器/last-only,无中途权重续训。新worker相对原worker只更换配置/首门模块并加fold3/4执行边界,AST检查确认其余训练科学逻辑全等。
|
||||
|
||||
新模型保留新合同SHA,三复用模型保留原训练合同SHA,canonical清单逐模型显式标origin与training_contract_sha256。只有两恢复折也PASS时才冻结完整五模型;任一失败只冻结available-models-incomplete,完整五折postfit门不得绕过。不把部分完成结果称为五折成功,也不自行给缺折科学评分。
|
||||
|
||||
## 新监控与失败隔离
|
||||
|
||||
仅新入口采用nvidia-smi timeout10秒,每一采样操作独立计连续失败数,成功有效采样清零该操作连续计数;第3次连续失败才触发该job的监控失败。每次错误都有时间、操作、异常、连续计数和原文记录。不同操作的成功不能掩盖持续GPU查询失败;缺采样标valid=false/null,不当成零占用或越限。
|
||||
|
||||
GPU状态查询和快速本地RSS/disk/已记录VRAM峰检查分线程。已实际观测的资源越限立即失败,不等3次;nvidia-smi等待不阻塞本地检查。内层进程supervisor亦在独立新副本使用同一连续失败规则,保留CPU/wait4/RSS峰、失败账与实际poll gap。允许的非连续/不足三次错误仍保留在monitor_errors和monitor_policy,不伪称零错误;回执PASS按显式policy.failed判断。证据无法写入、实际源码身份损坏或信号等不属于“缺一个样本就越限”的推断。
|
||||
|
||||
单job的数值/资源/三连监控失败或自身首轮/逐轮成本投影超帽只停止该job,另一折保持自身guard运行。累计资源/共享源码身份/明确operator停止仍是队列级保护。没有进一步恢复、换卡重试或重复派工。
|
||||
|
||||
## 卡占用与计费
|
||||
|
||||
只使用原分配GPU2/3:fold3→GPU3,fold4→GPU2。队列先实测idle,guard再取连续3次idle样本;有效busy样本保持pending,不杀占用者。GPU2于19:29:29左右快照为9MiB/0%,GPU3为约1.80GiB/14%,seed1原0/1/2任务仍在。此快照不是未来启动凭据;每次实际派发仍检查,seed1原队列/pins保持。
|
||||
|
||||
所有pending等待、setup与失败都计入对应恢复折;新单折保留原30960 GPU秒帽,其中60秒setup裕度和派发前等待不重计。guard用原帽扣掉派发前等待的实际余额;队列取独立墙钟与guard成本较大值。等待耗尽自身窗口只记录该折未派发失败,不挪动别的队列,也不无界免费等待。
|
||||
|
||||
旧训练47943.94719455298 GPU秒一次:旧+全部新训练/等待≤154800,postfit保留7200,总≤162000;新训练余额106856.05280544702。初始两恢复折保留61920,加post7200=69120,累计投影117063.94719455298。新入口不自动放宽单job30960或资源帽,15GiB VRAM/16GiB RSS/8GiB单fold产物保持,原协调器40GiB RSS/48GiB产物保持。旧guardCPU含worker不能相加,旧STOP无coordinatorCPU字段的缺口保持。新协调器在成功/失败终态记录自己的CPU;原始统计不是完整端到端CPU总数的伪造。
|
||||
|
||||
## 原 postfit 983bba99 的精确身份边界
|
||||
|
||||
原postfit实际PASS983bba99的科学方法和配对统计仍可沿用,但原入口固定训练root=`track-a-h2-2026-11-18`、旧合同/五新worker路径/全零监控错误规则。旧root已STOP且不改,因此原入口不能直接通过新恢复模型与容许的已记录瞬时错误。新训练入口不会混用旧PASS,不复制伪造五折成功receipt,也不会把新checkpoint重命名为原合同模型。
|
||||
|
||||
需要独立、仅身份适配的postfit新绑定,明确三原模型+两恢复模型/两合同身份、旧账一次、容许的采样错误与完整五折门;原评分/1537访问/586唯一件配对、原bootstrap保持。当前只记录兼容性限制,没有实现或启动postfit适配。恢复全五成功后的状态为TRAIN_COMPLETE_POSTFIT_REBIND_REQUIRED。若再次失败,只报告已完成折与失败清单,不擅自改原postfit为部分cohort评分。
|
||||
|
||||
## 合成检查与保留的失败
|
||||
|
||||
21项包括:10秒查询参数;两次失败后恢复/三连续失败/跨操作不可掩盖/缺样本非零;最终重采样有界;worker科学AST全等;旧账一次/失败成本保留/actual floor/非有限拒绝;缺折不得canonical五模型;容许错误不能被误判全零;从实际queue函数抽取执行,验证collect失败不停止另一折、忙卡不launch、三次派发查询失败只settle本折;自身成本门隔离;复用折禁止执行;快速本地检查不调用GPU查询;真实合成子进程内存越限立即停止、两次tree采样错误恢复、三次失败终止子进程。
|
||||
|
||||
首轮19项中1项失败原样保留。诊断显示被测试子进程return0、两个瞬时监控错误已恢复,但wait4峰RSS404328448B超过测试给的100000000B,reason=rss_limit_after_exit;不是监控容错逻辑未恢复。只将该合成测试及持续监控测试内存帽改为1GiB以隔离目标行为,另一个真实越限测试仍使用1B帽且必须失败。补齐两派发隔离测试后21项PASS。日志、原失败与诊断回执全部保留;没有科学数组/目标读取或GPU计算。
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "FIXED_PENDING_DELTA_REVIEW",
|
||||
"binding_sha256": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"code_files": 27,
|
||||
"source_files": 60,
|
||||
"config": "PASS",
|
||||
"reuse_identity": "PASS",
|
||||
"tests": 21,
|
||||
"tests_log_sha256": "7db6ca25fbc9586e893065728b955c72877f5f16dc43e455f1b83eeb9a576fdf",
|
||||
"old_failure_preserved": true,
|
||||
"postfit_binding_compatible": false,
|
||||
"started": false
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"approval": "1795000626.905989",
|
||||
"monitor_ruling": "1795000465.748129",
|
||||
"original_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"original_scientific_comparison_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"fresh_seed0_folds": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"epochs": 200,
|
||||
"last_epoch_only": true,
|
||||
"interrupted_weights_reused": false,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_gpu_cap": 162000,
|
||||
"new_train_cap": 106856.05280544702,
|
||||
"postfit_reserve": 7200,
|
||||
"per_new_job_cap": 30960,
|
||||
"monitoring": {
|
||||
"gpu_timeout_seconds": 10,
|
||||
"failed_samples": 3,
|
||||
"consecutive_per_operation": true,
|
||||
"all_failures_logged": true,
|
||||
"valid_sample_resets_own_streak": true,
|
||||
"missing_sample_is_not_breach": true,
|
||||
"observed_resource_breach_immediate": true,
|
||||
"fast_local_checks_independent_of_gpu_query": true,
|
||||
"worker_supervisor_tolerance_same_rule": true
|
||||
},
|
||||
"isolation": "Failed worker, query monitor, own projection or own idle cap affects only its fold; cumulative resource/source integrity/operator stop remains global",
|
||||
"idle": "Original assigned GPU2/3 only; actual idle sample then3guard idle samples; busy pending waits are charged per waiting fold inside own30960 and total budget; no interference with seed1",
|
||||
"no_further_recovery": true,
|
||||
"postfit": {
|
||||
"original_review_sha256": "983bba99d3ef5aac1476be4b579c9b3b29c5fbf94f1b3fd963b3e8c4977db556",
|
||||
"exact_binding_compatible": false,
|
||||
"reason": "Original postfit pins stopped old root and requires old five new workers. Mixed original/recovery checkpoints have two training-contract identities and tolerable logged sampling errors. Identity-only new postfit binding required before any test access; original statistics/pairing stay unchanged.",
|
||||
"partial_cohort_scoring_authorized": false,
|
||||
"auto_start": false
|
||||
}
|
||||
}
|
||||
+21422
File diff suppressed because it is too large.
Load diff
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"reviewed_commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"implementation_proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"comparison_contract_sha256": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5"
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2"
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/comparison-contract.json": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "7002cfc60511d80f6aae331d255fad501eeaaa32b63f1d7aa29b36e5b2558fae",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-entry-2026-11-18.md": "070536c0addeeef2d2ce70edb6387f26307fad5fc1cd0df22fec0def36398732",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "b5e937c870225059ec38fbefbb2b933ef46cea10fb38fe520156749da37cf3c8",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "7db6ca25fbc9586e893065728b955c72877f5f16dc43e455f1b83eeb9a576fdf",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "85483476d5bfa7274b5a3d658fe5e57227d26063b56a6827f6a9fb5960715b4d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "b967bd9e67c37f99af42b8160941dc5899a5193cc15bda243e2eaa79c5100434",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "4027923ffc3bd93b1b9cea875cbba236b56249e55a41af354cf19f226af6c441",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "c8aa937dec2e2ca9dfdaa102c0a61e4cb11b513d7ded085887b471e6d0a2478d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "805226f37c52f96a095566dd6ee5f1e4c20871ebada2db99af679c6d197458da"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/comparison-contract.json": "home-runs/track-a-recovery-2026-11-18/comparison-contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "home-runs/track-a-recovery-2026-11-18/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-entry-2026-11-18.md": "docs/track-a-recovery-entry-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py": "tools/prepare_track_a_recovery.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "tools/test_track_a_recovery.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "tools/track_a_recovery_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "tools/track_a_recovery_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "tools/track_a_recovery_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_NARROW_DELTA_REVIEW",
|
||||
"unix": 1795001387.8988416,
|
||||
"metadata_cpu_seconds": 4.274535771,
|
||||
"original_STOP_preserved": true,
|
||||
"reused_models_hashed": 3,
|
||||
"model_values_loaded": false,
|
||||
"scientific_arrays_decoded": false,
|
||||
"target_values_read": false,
|
||||
"gpu_query_performed": false,
|
||||
"training_started": false,
|
||||
"postfit_started": false,
|
||||
"code_files": 27,
|
||||
"source_files": 60,
|
||||
"initial_projection": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 30960
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 69120,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 117063.94719455298,
|
||||
"postfit_reserve": 7200
|
||||
}
|
||||
}
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
"""Metadata-only custody join, conservative components and deterministic S2 splits."""
|
||||
import argparse
|
||||
import collections
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import zipfile
|
||||
import numpy as np
|
||||
|
||||
BASE = Path('artifacts')
|
||||
OUT = BASE / 's2-g1-components-2026-11-15'
|
||||
REG = BASE / 's2-g1-registration-2026-11-14'
|
||||
BAND = BASE / 's2-g1-band25-2026-11-15'
|
||||
CMP = BASE / 's2-g1-band25-review-comparison-2026-11-15'
|
||||
CONTRACT = Path('docs/s2-g1-component-split-contract-2026-11-15.md')
|
||||
MERGE = 'MERGE_SAME_SIDE_LEAK_PROTECTION'
|
||||
NOEDGE = 'NO_ADDITIONAL_EDGE_BOTH_UNRELATED'
|
||||
EXPECTED = {
|
||||
str(REG/'batch-complete.json'): 'b0ee7469c1ab755e809f8ff85ebc6ce940e9838a23eb091a4a7a0e91e77d0956',
|
||||
str(REG/'contract.json'): 'c4271622e1775d430324dd8e8ad4524006184122079d2fc3bbb8fd0ba6a349d8',
|
||||
str(REG/'custodian/plan.json'): '61f4695b7fdf16d747bdebafc1d0c6d362087e250e9d85a36ccce3b3caa71af6',
|
||||
str(BAND/'custodian/mapping.json'): '7531566edf060cdae522b7cf32d0164d0cc6bd9d5b26be6a8d1c3a53bbf61142',
|
||||
str(CMP/'comparison.csv'): 'e7603e06ac5529608c23b889475632b5d75cfbeea7686528460a6c81dcd10bfd',
|
||||
str(CMP/'sources/mold/frozen-review.csv'): '6479eebdfc09629c7188f18f888494c5d9bea8dd160d92763813b15ef813ce18',
|
||||
str(CMP/'sources/claude/band25-review.csv'): '93096734186c1247e46e98f26c7c9b0f7a4987c8d8d588e8147110ae3880be2e',
|
||||
}
|
||||
|
||||
|
||||
def sha(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as f:
|
||||
for block in iter(lambda: f.read(1048576), b''):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def save(path, data):
|
||||
with Path(path).open('x') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
f.write('\n')
|
||||
|
||||
|
||||
class DSU:
|
||||
def __init__(self, members):
|
||||
self.parent = {x: x for x in members}
|
||||
|
||||
def root(self, x):
|
||||
while self.parent[x] != x:
|
||||
self.parent[x] = self.parent[self.parent[x]]
|
||||
x = self.parent[x]
|
||||
return x
|
||||
|
||||
def union(self, a, b):
|
||||
a, b = sorted((self.root(a), self.root(b)))
|
||||
self.parent[b] = a
|
||||
|
||||
|
||||
def action(a, b):
|
||||
allowed = {'unrelated', 'UNRESOLVED', 'rigid / same part'}
|
||||
assert a in allowed and b in allowed
|
||||
return NOEDGE if a == b == 'unrelated' else MERGE
|
||||
|
||||
|
||||
def order(ids, stage, fold):
|
||||
ids = sorted(ids)
|
||||
rng = np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111409, stage, fold])))
|
||||
return [ids[int(i)] for i in rng.permutation(len(ids))]
|
||||
|
||||
|
||||
def split_components(components):
|
||||
by_id = {c['component_id']: c for c in components}
|
||||
ids = sorted(by_id)
|
||||
rank = {x: i for i, x in enumerate(order(ids, 0, 0))}
|
||||
placement = sorted(ids, key=lambda x: (-by_id[x]['parts'], rank[x]))
|
||||
folds = [[] for _ in range(5)]
|
||||
parts = [0] * 5
|
||||
for x in placement:
|
||||
k = min(range(5), key=lambda k: (parts[k], len(folds[k]), k))
|
||||
folds[k].append(x)
|
||||
parts[k] += by_id[x]['parts']
|
||||
rows = []
|
||||
for k, test in enumerate(folds):
|
||||
remaining = set(ids) - set(test)
|
||||
val_order = order(remaining, 1, k)
|
||||
val = val_order[:math.ceil(len(remaining)/5)]
|
||||
learning = order(remaining-set(val), 2, k)
|
||||
pools = {str(p): learning[:n] for p, n in [(25, math.ceil(len(learning)/4)), (50, math.ceil(len(learning)/2)), (100, len(learning))]}
|
||||
reasons = []
|
||||
if len(ids) < 5: reasons.append('FEWER_THAN_FIVE_COMPONENTS')
|
||||
if not test: reasons.append('EMPTY_TEST_FOLD')
|
||||
if not val: reasons.append('EMPTY_VALIDATION_POOL')
|
||||
if not learning: reasons.append('EMPTY_LEARNING_POOL')
|
||||
if len({len(v) for v in pools.values()}) < 3: reasons.append('COINCIDENT_LEARNING_TIERS')
|
||||
assert not(set(test)&set(val) or set(test)&set(learning) or set(val)&set(learning))
|
||||
assert set(test)|set(val)|set(learning) == set(ids)
|
||||
assert pools['25'] == pools['50'][:len(pools['25'])]
|
||||
assert pools['50'] == pools['100'][:len(pools['50'])]
|
||||
rows.append(dict(fold=k, test=test, validation=val, validation_order=val_order,
|
||||
learning_order=learning, pools=pools, noncomparability_reasons=reasons))
|
||||
assert collections.Counter(x for f in folds for x in f) == collections.Counter(ids)
|
||||
return dict(order_version='g1-pcg64-seedsequence-v1', seed=2026111409,
|
||||
numpy_version=np.__version__, placement_order=placement, folds=rows)
|
||||
|
||||
|
||||
def components_stage():
|
||||
assert not OUT.exists(), 'Stage already exists: inspect rather than overwrite'
|
||||
for p, digest in EXPECTED.items(): assert sha(p) == digest, p
|
||||
for p, digest in read(CMP/'receipt.json')['source_sha256'].items():
|
||||
assert sha(CMP/p) == digest, p
|
||||
for p, digest in read(REG/'contract.json')['input_sha256'].items(): assert sha(p) == digest, p
|
||||
terminal = read(REG/'batch-complete.json')
|
||||
assert terminal['status'] == 'ALL_REGISTRATIONS_COMPLETE' and terminal['failed_jobs'] == 0
|
||||
plan = read(REG/'custodian/plan.json')
|
||||
assert len(terminal['jobs']) == len(plan['jobs']) == 2012
|
||||
assert {j['index'] for j in terminal['jobs']} == set(range(2012))
|
||||
assert all(j['returncode'] == 0 and j['worker']['status'] == 'PASS' and not j['worker']['retry'] for j in terminal['jobs'])
|
||||
inventory_path = BASE/'s2-g1-inventory-2026-11-14/custodian/parts.json'
|
||||
candidate_plan = BASE/'s2-g1-candidates-2026-11-14/custodian/plan.json'
|
||||
old_join = BASE/'s2-g1-custody-handoff-2026-11-14/custodian/joined-frozen-review.json'
|
||||
inventory = read(inventory_path)
|
||||
by_case = {r['case']: r for r in inventory}
|
||||
assert len(inventory) == len(by_case) == 624
|
||||
assert set(plan['geometry_cache']) == set(by_case)
|
||||
dev = set(read(candidate_plan)['development_cases'])
|
||||
old = read(old_join)['pairs']
|
||||
old_reviewed = {r[k] for r in old for k in ['a', 'b']}
|
||||
assert all(by_case[r['a']]['source_key'] == by_case[r['b']]['source_key'] for r in old)
|
||||
mapping = read(BAND/'custodian/mapping.json')
|
||||
rows = list(csv.DictReader((CMP/'comparison.csv').open()))
|
||||
original_fields = list(rows[0])
|
||||
assert len(rows) == 25 and len({r['anonymous_pair_id'] for r in rows}) == 25
|
||||
ids = {r['anonymous_pair_id'] for r in rows}
|
||||
assert ids == {p['anonymous_pair_id'] for p in mapping['pairs']}
|
||||
map_by_id = {p['anonymous_pair_id']: p for p in mapping['pairs']}
|
||||
for r in rows: r['final_action'] = action(r['mold_relation'], r['claude_relation'])
|
||||
assert {r['anonymous_pair_id'][:8] for r in rows if r['final_action'] == MERGE} == {'3058d0a6', '731fb76c', '1ead5efb', 'a7496efb'}
|
||||
OUT.mkdir()
|
||||
private = OUT/'custodian'; private.mkdir(mode=0o700)
|
||||
public = OUT/'public'; public.mkdir()
|
||||
shutil.copy2(CMP/'comparison.csv', public/'comparison-before-ruling.csv')
|
||||
with (public/'comparison.csv').open('x') as f:
|
||||
w = csv.DictWriter(f, original_fields+['final_action'], lineterminator='\n'); w.writeheader(); w.writerows(rows)
|
||||
reread = list(csv.DictReader((public/'comparison.csv').open()))
|
||||
assert [{k:r[k] for k in original_fields} for r in reread] == list(csv.DictReader((CMP/'comparison.csv').open()))
|
||||
shutil.copy2(CONTRACT, public/'execution-contract.md')
|
||||
input_sha = dict(EXPECTED)
|
||||
for p in [inventory_path, candidate_plan, old_join, CONTRACT, Path(__file__)]: input_sha[str(p)] = sha(p)
|
||||
source_groups = collections.defaultdict(list)
|
||||
dsu = DSU(by_case)
|
||||
edges = []
|
||||
for case, r in by_case.items(): source_groups[r['source_key']].append(case)
|
||||
assert len(source_groups) == 249
|
||||
for key, members in sorted(source_groups.items()):
|
||||
members.sort()
|
||||
for b in members[1:]:
|
||||
edges.append(dict(a=members[0], b=b, reason='SAME_SOURCE_KEY'))
|
||||
dsu.union(members[0], b)
|
||||
results = {}
|
||||
counts = collections.Counter()
|
||||
for i, pair in enumerate(plan['jobs']):
|
||||
path = REG/f'custodian/results/pair-{i:04d}.json'
|
||||
digest = sha(path); input_sha[str(path)] = digest
|
||||
r = read(path)
|
||||
assert r['index'] == i and r['pair'] == pair and r['simulation_fields_read'] is False
|
||||
status = r['engineering_rule']['engineering_status']; counts[status] += 1
|
||||
results[i] = r
|
||||
if status == 'LINK_UNDER_FIXED_RULE':
|
||||
edges.append(dict(a=pair['a'], b=pair['b'], reason=status, registration_index=i, result_sha256=digest))
|
||||
dsu.union(pair['a'], pair['b'])
|
||||
assert dict(counts) == {'OUTSIDE_FIXED_RULE':1897, 'BAND_REVIEW_REQUIRED':25, 'LINK_UNDER_FIXED_RULE':90}
|
||||
assert {p['registration_index'] for p in mapping['pairs']} == {i for i,r in results.items() if r['engineering_rule']['engineering_status']=='BAND_REVIEW_REQUIRED'}
|
||||
for path, digest in mapping['source_result_sha256'].items(): assert sha(path) == digest, path
|
||||
joined = []
|
||||
band_reviewed = set()
|
||||
for r in rows:
|
||||
m = map_by_id[r['anonymous_pair_id']]; i = m['registration_index']; pair = plan['jobs'][i]
|
||||
endpoints = {s['case'] for s in m['sides'].values()}
|
||||
assert endpoints == {pair['a'], pair['b']}
|
||||
band_reviewed.update(endpoints)
|
||||
joined.append(dict(**r, registration_index=i, a=pair['a'], b=pair['b'],
|
||||
engineering_rule=results[i]['engineering_rule'], result_sha256=input_sha[str(REG/f'custodian/results/pair-{i:04d}.json')]))
|
||||
if r['final_action'] == MERGE:
|
||||
edges.append(dict(a=pair['a'], b=pair['b'], reason=MERGE, anonymous_pair_id=r['anonymous_pair_id'], registration_index=i))
|
||||
dsu.union(pair['a'], pair['b'])
|
||||
nodes = {}; node_sources = {}; exposure = {}
|
||||
for case in sorted(by_case):
|
||||
cache = plan['geometry_cache'][case]; path = Path(cache['path'])
|
||||
assert sha(path) == cache['sha256'], case
|
||||
with zipfile.ZipFile(path) as z:
|
||||
info = z.getinfo('points.npy')
|
||||
with z.open(info) as f:
|
||||
version = np.lib.format.read_magic(f)
|
||||
assert version == (1,0)
|
||||
shape, fortran, dtype = np.lib.format.read_array_header_1_0(f)
|
||||
header_bytes = f.tell()
|
||||
assert len(shape) == 2 and shape[1] == 3 and shape[0] > 0 and not dtype.hasobject
|
||||
assert info.file_size == header_bytes + math.prod(shape)*dtype.itemsize
|
||||
nodes[case] = int(shape[0])
|
||||
node_sources[case] = dict(**cache, points_shape=shape, points_dtype=str(dtype), header_bytes=header_bytes,
|
||||
points_crc32=info.CRC, points_uncompressed_bytes=info.file_size, payload_deserialized=False)
|
||||
inv = by_case[case]
|
||||
exposure[case] = dict(inventory_state=inv['exposure_state'], old_30_pair_endpoint=inv['old_30_pair_endpoint'],
|
||||
individual_diagnostic_stages=inv['individual_diagnostic_stages'], calibration_development=case in dev,
|
||||
blind120_geometry_review=case in old_reviewed, band25_geometry_review=case in band_reviewed,
|
||||
all_corpus_historical_screening=inv['all_corpus_historical_screening'], unexposed_claim=False,
|
||||
development_exposed=inv['exposure_state']=='KNOWN_DEVELOPMENT_EXPOSED' or case in dev)
|
||||
groups = collections.defaultdict(list)
|
||||
for case in sorted(by_case): groups[dsu.root(case)].append(case)
|
||||
components = []; private_components = []; case_component = {}
|
||||
for members in groups.values():
|
||||
cid = 'component-'+hashlib.sha256('\n'.join(members).encode()).hexdigest()[:16]
|
||||
record = dict(component_id=cid, parts=len(members), nodes=sum(nodes[c] for c in members),
|
||||
source_keys=len({by_case[c]['source_key'] for c in members}),
|
||||
development_exposed_parts=sum(exposure[c]['development_exposed'] for c in members),
|
||||
blind120_geometry_review_parts=sum(c in old_reviewed for c in members),
|
||||
band25_geometry_review_parts=sum(c in band_reviewed for c in members), unexposed_claim=False)
|
||||
components.append(record); private_components.append(dict(**record, members=members))
|
||||
case_component.update({c:cid for c in members})
|
||||
components.sort(key=lambda c:c['component_id'])
|
||||
assert len({c['component_id'] for c in components}) == len(components)
|
||||
assert all(case_component[e['a']] == case_component[e['b']] for e in edges)
|
||||
key_public = []; key_private = []
|
||||
for i,(key, members) in enumerate(sorted(source_groups.items())):
|
||||
kid = f'key-{i:03d}'
|
||||
row = dict(key_id=kid, parts=len(members), nodes=sum(nodes[c] for c in members), component_id=case_component[members[0]])
|
||||
key_public.append(row); key_private.append(dict(**row, source_key=key, members=members))
|
||||
total_nodes = sum(nodes.values())
|
||||
largest = max(components, key=lambda c:(c['parts'], c['nodes'], c['component_id']))
|
||||
largest_nodes = max(components, key=lambda c:(c['nodes'], c['parts'], c['component_id']))
|
||||
def shares(c): return dict(**c, part_share=c['parts']/624, node_share=c['nodes']/total_nodes)
|
||||
flags = [shares(c) for c in components if c['parts']*5 > 624 or c['nodes']*5 > total_nodes]
|
||||
stats = dict(status='COMPONENTS_FROZEN_SPLITS_NOT_YET_RUN', parts=624, source_keys=249, nodes=total_nodes,
|
||||
components=len(components), component_size_distribution=dict(sorted(collections.Counter(c['parts'] for c in components).items())),
|
||||
source_key_size_distribution=dict(sorted(collections.Counter(len(m) for m in source_groups.values()).items())),
|
||||
largest_by_parts=shares(largest), largest_by_nodes=shares(largest_nodes), oversized_report_flags=flags,
|
||||
edge_counts=dict(collections.Counter(e['reason'] for e in edges)), registration_status_counts=dict(counts),
|
||||
band_final_action_counts=dict(collections.Counter(r['final_action'] for r in rows)),
|
||||
development_exposed_parts=sum(x['development_exposed'] for x in exposure.values()),
|
||||
development_exposed_components=sum(c['development_exposed_parts']>0 for c in components),
|
||||
labels_read=False, geometry_recomputed=False, equivalence_truth=False, cross_key_recall_certified=False,
|
||||
ruling_message_ts='1794678062.264009', node_definition='Unique nodes used by tetrahedra; existing points.npy shape',
|
||||
node_cache_sha256_all_verified=True, original_comparison_columns_preserved=True)
|
||||
save(private/'joined-band-review.json', joined)
|
||||
save(private/'edges.json', edges)
|
||||
save(private/'components.json', private_components)
|
||||
save(private/'source-keys.json', key_private)
|
||||
save(private/'parts.json', [dict(case=c, source_key=by_case[c]['source_key'], component_id=case_component[c], nodes=nodes[c], exposure=exposure[c]) for c in sorted(by_case)])
|
||||
save(private/'node-metadata-provenance.json', node_sources)
|
||||
save(private/'input-sha256.json', input_sha)
|
||||
save(public/'components.json', components)
|
||||
save(public/'parts-per-key.json', key_public)
|
||||
save(public/'component-statistics.json', stats)
|
||||
receipt = dict(status='COMPONENTS_FROZEN', ruling_message_ts='1794678062.264009', python=platform.python_version(), numpy=np.__version__,
|
||||
source_sha256=EXPECTED, contract_sha256=sha(CONTRACT), code_sha256=sha(__file__),
|
||||
custody_artifact_sha256={p.name:sha(p) for p in sorted(private.iterdir())},
|
||||
public_artifact_sha256={p.name:sha(p) for p in sorted(public.iterdir())},
|
||||
reviewer_mapping_returned=False, original_source_files_unchanged=all(sha(p)==h for p,h in EXPECTED.items()))
|
||||
save(public/'component-receipt.json', receipt)
|
||||
for p in private.iterdir(): p.chmod(0o600)
|
||||
print(json.dumps(stats, ensure_ascii=False))
|
||||
|
||||
|
||||
def split_stage(report_ts):
|
||||
public = OUT/'public'; private = OUT/'custodian'
|
||||
assert not (public/'splits.json').exists(), 'Splits already exist'
|
||||
stats = read(public/'component-statistics.json')
|
||||
assert not stats['oversized_report_flags'] or report_ts, 'Report oversized components before splitting'
|
||||
receipt = read(public/'component-receipt.json')
|
||||
assert sha(CONTRACT) == receipt['contract_sha256'] and sha(__file__) == receipt['code_sha256']
|
||||
for name, digest in receipt['custody_artifact_sha256'].items(): assert sha(private/name) == digest
|
||||
for name, digest in receipt['public_artifact_sha256'].items(): assert sha(public/name) == digest
|
||||
components = read(public/'components.json'); by_id = {c['component_id']:c for c in components}
|
||||
splits = split_components(components)
|
||||
def summarize(ids):
|
||||
return dict(components=len(ids), parts=sum(by_id[c]['parts'] for c in ids), nodes=sum(by_id[c]['nodes'] for c in ids),
|
||||
development_exposed_parts=sum(by_id[c]['development_exposed_parts'] for c in ids),
|
||||
development_exposed_components=sum(by_id[c]['development_exposed_parts']>0 for c in ids), unexposed_claim=False)
|
||||
for fold in splits['folds']:
|
||||
fold['counts'] = dict(test=summarize(fold['test']), validation=summarize(fold['validation']),
|
||||
pools={p:summarize(ids) for p,ids in fold['pools'].items()})
|
||||
save(public/'splits.json', splits)
|
||||
private_rows = []
|
||||
for part in read(private/'parts.json'):
|
||||
cid = part['component_id']; assignments = []
|
||||
for f in splits['folds']:
|
||||
role = 'test' if cid in f['test'] else 'validation' if cid in f['validation'] else 'learning'
|
||||
assignments.append(dict(fold=f['fold'], role=role, learning_pools=[p for p,ids in f['pools'].items() if cid in ids]))
|
||||
private_rows.append(dict(**part, assignments=assignments))
|
||||
save(private/'part-splits.json', private_rows)
|
||||
(private/'part-splits.json').chmod(0o600)
|
||||
result = dict(status='COMPONENTS_AND_FIVE_FOLD_NESTED_POOLS_FROZEN_AWAITING_REVIEW',
|
||||
contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), components_receipt_sha256=sha(public/'component-receipt.json'),
|
||||
splits_sha256=sha(public/'splits.json'), custody_part_splits_sha256=sha(private/'part-splits.json'),
|
||||
oversized_components_report_message_ts=report_ts, no_component_cut=True,
|
||||
all_required_edges_same_fold=True, component_test_partition_verified=True, nested_pools_verified=True,
|
||||
no_geometry_registration_review_rerun=True, training_pilot_gpu=False,
|
||||
fold_counts=[dict(fold=f['fold'], **f['counts'], noncomparability_reasons=f['noncomparability_reasons']) for f in splits['folds']])
|
||||
save(public/'split-receipt.json', result)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser(); p.add_argument('stage', choices=['components','splits']); p.add_argument('--oversized-report-ts')
|
||||
args = p.parse_args()
|
||||
if args.stage == 'components': components_stage()
|
||||
else: split_stage(args.oversized_report_ts)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
|
||||
import numpy as np
|
||||
|
||||
|
||||
def xgb_model(seed,threads=8):
|
||||
from xgboost import XGBRegressor
|
||||
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
|
||||
objective='reg:squarederror',random_state=seed,n_jobs=threads)
|
||||
|
||||
|
||||
def fnn_model():
|
||||
import torch
|
||||
layers=[];width=19
|
||||
for _ in range(4):
|
||||
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
|
||||
layers.append(torch.nn.Linear(100,1))
|
||||
return torch.nn.Sequential(*layers)
|
||||
|
||||
|
||||
def eligible_roles(split,successful):
|
||||
successful=set(successful)
|
||||
roles={role:[c for c in split[role] if c in successful]
|
||||
for role in ['train','test','fnn_learning','fnn_validation']}
|
||||
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
|
||||
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
|
||||
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
|
||||
raise ValueError('FNN learning/validation do not partition train')
|
||||
missing={role:[c for c in split[role] if c not in successful] for role in roles}
|
||||
return roles,missing
|
||||
|
||||
|
||||
def require_training_role(case,split,role):
|
||||
if role not in ['train','fnn_learning','fnn_validation']:
|
||||
raise PermissionError('Only frozen training-side roles may read targets')
|
||||
if case in split['test'] or case not in split[role]:
|
||||
raise PermissionError('Case is not in the requested training role')
|
||||
|
||||
|
||||
def early_stop_update(best,stale,score,patience=5):
|
||||
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
|
||||
improved=score<best
|
||||
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
"""Exact semantic PASS and immutable source/contract identity before custody."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import sha,checked
|
||||
from phase2_home_freeze_io import verify
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def validate_review(root):
|
||||
root=Path(root);c=read(root/'contract.json');proof=verify(root,'implementation');binding=read(root/'expected-review-binding.json');review=read(root/'review.json')
|
||||
expected=dict(reviewed_commit=proof['commit'],contract_sha256=sha(root/'contract.json'),implementation_proof_sha256=sha(root/'implementation-git-freeze.json'),comparison_contract_sha256=c['comparison_contract_sha256'],code_sha256=c['code_sha256'])
|
||||
if binding!=expected:raise PermissionError('Review binding differs from implementation')
|
||||
if review.get('verdict')!='PASS':raise PermissionError('Actual exact PASS required')
|
||||
for name,value in expected.items():
|
||||
if review.get(name)!=value:raise PermissionError('Review identity mismatch: '+name)
|
||||
checked(c['comparison_contract'],expected['comparison_contract_sha256'])
|
||||
for p,h in expected['code_sha256'].items():checked(p,h)
|
||||
frozen=verify(root,'implementation-review')
|
||||
for p in [root/'review.json',root/'expected-review-binding.json',root/'contract.json']:
|
||||
if frozen['files'].get(str(p.resolve()))!=sha(p):raise PermissionError('Review freeze does not include exact required file')
|
||||
return review
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
"""H1-RES fixed-XGB worker. No queue or implicit execution approval.
|
||||
|
||||
The reviewed contract binds training/test roles, source metadata, precision,
|
||||
clock weighting and source hashes. Test predictions never load target arrays.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import numpy as np
|
||||
from threadpoolctl import threadpool_limits
|
||||
from gre24_models import xgb_model
|
||||
from phase2_residual_math import learning_clock, residual_target, decode_seconds
|
||||
|
||||
|
||||
def sha(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as source:
|
||||
for block in iter(lambda: source.read(4*1024**2), b''): h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def save(path, value):
|
||||
path = Path(path); tmp = path.with_suffix(path.suffix+'.tmp')
|
||||
tmp.write_text(json.dumps(value, indent=2, allow_nan=False)+'\n'); tmp.replace(path)
|
||||
|
||||
|
||||
def checked(path, digest):
|
||||
if sha(path) != digest: raise ValueError('Pinned file mismatch: '+str(path))
|
||||
return Path(path)
|
||||
|
||||
|
||||
def contract(root):
|
||||
import xgboost
|
||||
if np.__version__ != '1.26.4' or xgboost.__version__ != '2.1.4':
|
||||
raise ValueError('Frozen library version differs')
|
||||
c = json.loads((root/'contract.json').read_text())
|
||||
if c.get('version') != 'phase2-h1-res-v1' or not c.get('main_review'):
|
||||
raise PermissionError('No fixed reviewed H1 contract')
|
||||
for p, h in c['code_sha256'].items(): checked(p, h)
|
||||
for p, h in c['source_sha256'].items(): checked(p, h)
|
||||
return c
|
||||
|
||||
|
||||
def role_rows(c, job, role):
|
||||
if role not in ('learning', 'test'): raise ValueError('Unknown role')
|
||||
if set(job['learning']) & set(job['test']): raise PermissionError('Training/test overlap')
|
||||
if len(job[role]) != len(set(job[role])): raise ValueError('Repeated case')
|
||||
# Contract builder must cross-check these frozen roles against original
|
||||
# source contracts. Here a role cannot be expanded through row metadata.
|
||||
return [c['parts'][job['protocol']][case] for case in job[role]]
|
||||
|
||||
|
||||
def load_features(row):
|
||||
source = Path(row['output']); hashes = row['output_sha256']
|
||||
f = np.load(checked(source/'Vf.npy', hashes['Vf.npy']), allow_pickle=False)
|
||||
x = np.load(checked(source/'features.npy', hashes['features.npy']), mmap_mode='r', allow_pickle=False)
|
||||
ids = np.load(checked(source/'raw_ids.npy', hashes['raw_ids.npy']), allow_pickle=False)
|
||||
if x.shape != (row['nodes'], 19) or f.shape != (row['nodes'],) or ids.shape != f.shape:
|
||||
raise ValueError('Feature/node identity shape mismatch')
|
||||
if f.dtype != np.dtype('float64') or not np.array_equal(np.asarray(x[:, 0],dtype=np.float32), f.astype(np.float32)):
|
||||
raise ValueError('Original float64 F differs from frozen first column')
|
||||
if np.any(np.diff(ids) <= 0) or not np.isfinite(x).all(): raise ValueError('Bad IDs/features')
|
||||
return np.asarray(x, dtype=np.float32), f, ids
|
||||
|
||||
|
||||
def fit(root, c, job):
|
||||
out = root/'fits'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False)
|
||||
save(out/'started.json', {'unix': time.time(), 'job': job, 'contract_sha256': sha(root/'contract.json')})
|
||||
cpu = time.process_time(); access = []; rows = role_rows(c, job, 'learning')
|
||||
end_ratios, groups = [], []
|
||||
# First pass: only learning-role retained-tet targets for this cell's clock.
|
||||
for row in rows:
|
||||
path = Path(row['output'])/'seconds.npy'
|
||||
access.append({'case': row['case'], 'role': 'learning', 'purpose': 'clock', 'path': str(path)})
|
||||
save(out/'target-access.json', access)
|
||||
y = np.load(checked(path, row['output_sha256']['seconds.npy']), mmap_mode='r', allow_pickle=False)
|
||||
if y.shape != (row['nodes'],) or not np.isfinite(y).all() or np.any(y < 0): raise ValueError('Invalid learning target')
|
||||
if y.dtype != np.dtype('float64') or not np.isfinite(row['t_set']) or row['t_set']<=0: raise ValueError('Endpoint precision/scale mismatch')
|
||||
raw=np.load(checked(Path(row['output'])/'raw_ids.npy',row['output_sha256']['raw_ids.npy']),allow_pickle=False)
|
||||
if raw.shape!=y.shape or np.any(np.diff(raw)<=0): raise ValueError('Endpoint raw ID mismatch')
|
||||
end_ratios.append(float(y.max())/float(row['t_set'])); groups.append(row['clock_group'])
|
||||
clock = learning_clock(end_ratios, groups)
|
||||
save(out/'clock.json', {'c': clock, 'source': 'learning retained-tet max(T)/t_set, mean within component then mean across components',
|
||||
'parts': [row['case'] for row in rows], 'end_ratios': end_ratios, 'groups': groups})
|
||||
n = sum(row['nodes'] for row in rows); x = np.empty((n, 19), np.float32); y = np.empty(n, np.float32); cursor = 0
|
||||
for row in rows:
|
||||
xx, f, ids = load_features(row)
|
||||
path = checked(Path(row['output'])/'target.npy', row['output_sha256']['target.npy'])
|
||||
target = np.load(path, mmap_mode='r', allow_pickle=False)
|
||||
physical=np.load(checked(Path(row['output'])/'seconds.npy',row['output_sha256']['seconds.npy']),mmap_mode='r',allow_pickle=False)
|
||||
if not np.array_equal(target,physical/row['t_set']): raise ValueError('Frozen normalized target differs from physical seconds/t_set')
|
||||
count = len(f); x[cursor:cursor+count] = xx
|
||||
y[cursor:cursor+count] = residual_target(target, f, clock, fit_dtype='float32'); cursor += count
|
||||
access.append({'case': row['case'], 'role': 'learning', 'purpose': 'residual_fit', 'path': str(path)})
|
||||
save(out/'target-access.json', access)
|
||||
with threadpool_limits(job['threads']):
|
||||
model = xgb_model(job['seed'], job['threads'])
|
||||
save(out/'progress.json', {'stage': 'FIT', 'nodes': n, 'clock': clock})
|
||||
model.fit(x, y, verbose=False)
|
||||
checkpoint = out/'model.ubj'; model.save_model(checkpoint)
|
||||
(out/'xgboost-config.json').write_text(model.get_booster().save_config())
|
||||
save(out/'result.json', {'status': 'FIT_COMPLETE_NO_TEST_SCORES', 'checkpoint': str(checkpoint),
|
||||
'checkpoint_sha256': sha(checkpoint), 'clock_sha256': sha(out/'clock.json'), 'c': clock,
|
||||
'learning_nodes': n, 'learning_parts': len(rows), 'cpu_seconds': time.process_time()-cpu,
|
||||
'test_labels_read': False, 'new_decodes': 0, 'retry': False})
|
||||
|
||||
|
||||
def predict(root, c, job):
|
||||
import joblib
|
||||
import xgboost
|
||||
proof = json.loads((root/'models-git-freeze.json').read_text())
|
||||
if not proof.get('commit') or not proof.get('checkout'):
|
||||
raise PermissionError('All-terminal model Git freeze missing or mismatched')
|
||||
for rel in ['contract.json', 'batch-complete.json', 'models-terminal.json']:
|
||||
path = root/rel; expected = proof['sha256'][rel]; checked(path, expected)
|
||||
repository_path = str(path.resolve().relative_to(Path.cwd()))
|
||||
raw = subprocess.check_output(['git', 'show', proof['commit']+':'+repository_path], cwd=proof['checkout'])
|
||||
if hashlib.sha256(raw).hexdigest() != expected: raise PermissionError('Git freeze differs from local files')
|
||||
terminal = json.loads((root/'batch-complete.json').read_text())
|
||||
if terminal['global_stop'] or not terminal['accounting_complete'] or terminal['attempted'] != len(c['jobs']):
|
||||
raise PermissionError('Fit queue not complete')
|
||||
freeze = json.loads((root/'models-terminal.json').read_text())
|
||||
if len(freeze['models']) != len(c['jobs']): raise PermissionError('Incomplete model manifest')
|
||||
entry = next(r for r in freeze['models'] if r['index'] == job['index'])
|
||||
if entry['status'] != 'PASS': raise ValueError('No checkpoint for failed cell')
|
||||
out = root/'predictions'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False)
|
||||
model = xgboost.XGBRegressor(); model.load_model(checked(entry['checkpoint'], entry['checkpoint_sha256']))
|
||||
old = job['baseline_model']; checked(old['path'], old['sha256'])
|
||||
if old['format'] == 'joblib':
|
||||
baseline = joblib.load(old['path'])
|
||||
elif old['format'] == 'ubj':
|
||||
baseline = xgboost.XGBRegressor(); baseline.load_model(old['path'])
|
||||
else:
|
||||
raise ValueError('Unsupported frozen A checkpoint format')
|
||||
clock = json.loads(checked(entry['clock_path'], entry['clock_sha256']).read_text())['c']
|
||||
index = []; cpu = time.process_time()
|
||||
with threadpool_limits(job['threads']):
|
||||
for row in role_rows(c, job, 'test'):
|
||||
x, f, ids = load_features(row) # No target.npy load on this path.
|
||||
seconds = decode_seconds(model.predict(x), f, clock, row['t_set'])
|
||||
old_seconds = np.asarray(baseline.predict(x), dtype=np.float64)*row['t_set']
|
||||
if old_seconds.shape != seconds.shape or not np.isfinite(old_seconds).all(): raise ValueError('Invalid A prediction')
|
||||
if job['protocol']=='grouped':
|
||||
prior=c['baseline_predictions'][str(job['index'])][row['case']]
|
||||
with np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as z:
|
||||
iq=np.searchsorted(ids,z['raw_ids'])
|
||||
if np.any(iq>=len(ids)) or not np.array_equal(ids[iq],z['raw_ids']):raise ValueError('Historical A query identity differs')
|
||||
if not np.array_equal(old_seconds[iq],z[prior['array']]):raise ValueError('Historical A query prediction differs')
|
||||
path = out/(row['case']+'.npz')
|
||||
np.savez_compressed(path, seconds=seconds, A_seconds=old_seconds, cF_seconds=clock*f*row['t_set'],
|
||||
EIK_seconds=f*row['t_set'], raw_ids=ids)
|
||||
index.append({'case': row['case'], 'path': str(path), 'sha256': sha(path),
|
||||
'negative_predictions': int(np.sum(seconds < 0)), 'nodes': len(ids)})
|
||||
save(out/'progress.json', {'parts': len(index), 'cpu_seconds': time.process_time()-cpu})
|
||||
save(out/'index.json', index)
|
||||
save(out/'result.json', {'status': 'PREDICTIONS_COMPLETE_NO_SCORES', 'index_sha256': sha(out/'index.json'),
|
||||
'cpu_seconds': time.process_time()-cpu, 'target_values_read': False})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
|
||||
parser.add_argument('mode', choices=['fit', 'predict']); parser.add_argument('index', type=int)
|
||||
args = parser.parse_args(); start = time.process_time()
|
||||
try:
|
||||
config = contract(args.root); selected = config['jobs'][args.index]
|
||||
if selected['index'] != args.index: raise ValueError('Job index mismatch')
|
||||
(fit if args.mode == 'fit' else predict)(args.root, config, selected)
|
||||
except Exception as exc:
|
||||
dest = args.root/'failures'; dest.mkdir(exist_ok=True)
|
||||
save(dest/f'{args.mode}-{args.index:02d}.json', {'type': type(exc).__name__, 'message': str(exc),
|
||||
'traceback': traceback.format_exc(), 'pid': os.getpid(), 'cpu_seconds': time.process_time()-start,
|
||||
'retry': False})
|
||||
raise
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
"""Frozen H1 interface and label-scoped H2 inputs; no test target access."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from scipy.sparse import csr_matrix
|
||||
from scipy.sparse.csgraph import connected_components
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase2_h2_model import edge_attributes
|
||||
from phase2_residual_math import residual_target
|
||||
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794900702.867059' or c['mode']!='PILOT_ONLY':raise PermissionError('No approved pilot contract')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
||||
if torch.__version__!=c['versions']['torch'] or np.__version__!=c['versions']['numpy']:raise ValueError('Library identity changed')
|
||||
return c
|
||||
|
||||
def cached(c,case,name):
|
||||
r=c['parts'][case];return np.load(checked(Path(r['output'])/(name+'.npy'),r['output_sha256'][name+'.npy']),allow_pickle=False)
|
||||
|
||||
def standardization(c,root):
|
||||
n=0;mean=np.zeros(19,np.float64);M2=np.zeros(19,np.float64);seen=[];start=time.process_time()
|
||||
for case in c['folds'][0]['learning']:
|
||||
x=np.asarray(cached(c,case,'features'),np.float64)
|
||||
if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid')
|
||||
k=len(x);avg=x.mean(0);delta=avg-mean;batch=np.square(x-avg).sum(0)
|
||||
M2+=batch+delta*delta*(n*k/(n+k));mean+=delta*(k/(n+k));n+=k;seen.append(case)
|
||||
save(root/'progress.json',dict(stage='LEARNING_ONLY_STANDARDIZATION',parts=len(seen),nodes=n,cpu_seconds=time.process_time()-start,labels_read=False))
|
||||
std=np.sqrt(M2/n);scale=np.where(std==0,1.,std)
|
||||
if not np.isfinite(scale).all() or n!=c['folds'][0]['learning_nodes']:raise ValueError('Standardization coverage')
|
||||
out=dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),nodes=n,parts=seen,labels_read=False,cpu_seconds=time.process_time()-start)
|
||||
save(root/'standardization.json',out);return out
|
||||
|
||||
def load_part(c,case,stats,role,root):
|
||||
allowed=c['folds'][0][role] if role in ('learning','validation') else []
|
||||
if case not in allowed:raise PermissionError('Pilot role violation')
|
||||
r=c['parts'][case];g=c['geometry'][case]
|
||||
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:
|
||||
points=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy()
|
||||
indptr=z['indptr'].copy();sender=z['indices'].astype(np.int64);old_rg=float(z['rg'])
|
||||
n=len(raw);receiver=np.repeat(np.arange(n,dtype=np.int64),np.diff(indptr))
|
||||
if not np.array_equal(raw,cached(c,case,'raw_ids')) or not np.array_equal(mass,cached(c,case,'mass')):raise ValueError('Frozen geometry/node mass identity')
|
||||
if n!=r['nodes'] or len(sender)!=g['directed_edges']:raise ValueError('Frozen graph counts')
|
||||
a,rg=edge_attributes(points,mass,receiver,sender)
|
||||
if not np.isclose(rg,old_rg,rtol=1e-10,atol=1e-12):raise ValueError('Original mass Rg differs')
|
||||
graph=csr_matrix((np.ones(len(sender)),sender,indptr),shape=(n,n));count,comp=connected_components(graph,directed=False)
|
||||
x=cached(c,case,'features');F=cached(c,case,'Vf')
|
||||
if F.dtype!=np.float64 or F.shape!=(n,) or not np.isfinite(F).all():raise ValueError('Original F must be FP64')
|
||||
x=((np.asarray(x,np.float64)-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
||||
if not np.isfinite(x).all():raise ValueError('Standardized feature overflow')
|
||||
result=dict(x=torch.from_numpy(x),mass=torch.from_numpy(mass.astype(np.float32)),receiver=torch.from_numpy(receiver),sender=torch.from_numpy(sender),attributes=torch.from_numpy(a),component_ids=torch.from_numpy(comp.astype(np.int64)),
|
||||
F=F,raw_ids=raw,physical_components=int(count),nodes=n,directed_edges=len(sender),Rg=rg,t_set=r['t_set'])
|
||||
if role=='learning':
|
||||
# Append the intended role/hash before opening any cached target.
|
||||
with (root/'learning-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,role=role,target_sha256=r['output_sha256']['target.npy'],clock_sha256=c['folds'][0]['clock_sha256']))+'\n')
|
||||
y=cached(c,case,'target');result['target']=torch.from_numpy(residual_target(y,F,c['folds'][0]['c'],fit_dtype='float32'))
|
||||
return result
|
||||
|
||||
def to_device(data,device):
|
||||
return {k:(v.to(device) if isinstance(v,torch.Tensor) else v) for k,v in data.items()}
|
||||
|
||||
def arguments(data):return tuple(data[k] for k in ['x','mass','receiver','sender','attributes','component_ids'])
|
||||
|
||||
def projection(c,epoch_seconds,validation_seconds,pilot_seconds):
|
||||
f0=c['folds'][0];per_fold=[]
|
||||
for f in c['folds']:
|
||||
learning_ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],len(f['learning'])/len(f0['learning']))
|
||||
validation_ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation']))
|
||||
prediction_ratio=max(f['test_nodes']/f0['validation_nodes'],f['test_edges']/f0['validation_edges'],len(f['test'])/len(f0['validation']))
|
||||
per_fold.append(dict(fold=f['fold'],train_200_seconds=200*epoch_seconds*learning_ratio,validation_20_seconds=20*validation_seconds*validation_ratio,test_prediction_seconds=validation_seconds*prediction_ratio,
|
||||
learning_ratio=learning_ratio,validation_ratio=validation_ratio,test_ratio=prediction_ratio))
|
||||
# Every measured forward/backward time includes load, transfers and optimizer.
|
||||
training=1.25*sum(f['train_200_seconds']+f['validation_20_seconds'] for f in per_fold)
|
||||
prediction=1.25*sum(f['test_prediction_seconds'] for f in per_fold)
|
||||
total=pilot_seconds+training+prediction+c['future_setup_reserve_seconds']
|
||||
passed=training<=12*3600 and prediction<=2*3600 and total<=16*3600
|
||||
return dict(per_fold=per_fold,measured_full_epoch_seconds=epoch_seconds,measured_validation_inference_seconds=validation_seconds,
|
||||
safety_factor=1.25,training_with_validation_seconds=training,prediction_seconds=prediction,pilot_seconds=pilot_seconds,future_setup_seconds=c['future_setup_reserve_seconds'],total_gpu_seconds=total,
|
||||
fits_12h_training_2h_prediction_16h_total=passed,decision='COST_FEASIBLE_FULL_SCOPE_AUTHORIZED' if passed else 'PARKED_RESOURCE',
|
||||
full_run_authorized_by='1794900702.867059',full_run_started=False,epoch_target=200,folds=5,new_fits=5,validation_targets_read=False,
|
||||
limitation='Validation forward timing only; no target metrics in pilot. CPU final field/topology scoring is unmeasured and cannot be claimed affordable from this GPU projection.')
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""H2-GEO reviewed fixed FP32 graph/token recipe; no data or launches."""
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
|
||||
EDGE_CHUNK=65536
|
||||
NODE_CHUNK=8192
|
||||
ATOL=3e-5
|
||||
RTOL=3e-4
|
||||
|
||||
def lr_epoch(epoch):
|
||||
if not 1<=epoch<=200:raise ValueError('Fixed 1-based epochs 1..200')
|
||||
if epoch<=10:return 1e-4+(epoch-1)*9e-4/9
|
||||
return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(epoch-10)/190))/2
|
||||
|
||||
def namespace_components(part_ids,local_ids):
|
||||
keys=list(zip(part_ids,local_ids));lookup={k:i for i,k in enumerate(sorted(set(keys)))}
|
||||
return np.array([lookup[k] for k in keys],np.int64)
|
||||
|
||||
def edge_attributes(points,mass,receiver,sender):
|
||||
x=np.asarray(points,np.float64);m=np.asarray(mass,np.float64)
|
||||
if not np.isfinite(x).all() or not np.isfinite(m).all() or np.any(m<=0):raise ValueError('Invalid geometry/mass')
|
||||
center=np.sum(x*m[:,None],axis=0)/m.sum()
|
||||
rg=float(np.sqrt(np.sum(m*np.sum((x-center)**2,axis=1))/m.sum()))
|
||||
if not math.isfinite(rg) or rg<=0:raise ValueError('Invalid mass Rg')
|
||||
xi=(x-center)/rg;delta=xi[sender]-xi[receiver];length=np.linalg.norm(delta,axis=1)
|
||||
if np.any(length<=0):raise ValueError('Nonpositive true edge')
|
||||
return np.column_stack([delta,length]).astype(np.float32),rg
|
||||
|
||||
class GeoBlock(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.local_norm=nn.LayerNorm(64);self.global_norm=nn.LayerNorm(64);self.ffn_norm=nn.LayerNorm(64)
|
||||
self.edge=nn.Sequential(nn.Linear(4,16),nn.GELU(),nn.Linear(16,1),nn.Softplus())
|
||||
self.local_value=nn.Linear(64,64,bias=False)
|
||||
self.assignment=nn.Linear(64,64);self.global_value=nn.Linear(64,64,bias=False)
|
||||
self.attention=nn.MultiheadAttention(64,4,dropout=0,batch_first=True)
|
||||
self.ffn=nn.Sequential(nn.Linear(64,128),nn.GELU(),nn.Linear(128,64))
|
||||
|
||||
def local(self,h,m,receiver,sender,attributes,edge_chunk):
|
||||
values=self.local_value(h);num=h.new_zeros(h.shape);den=h.new_zeros((len(h),1))
|
||||
for lo in range(0,len(sender),edge_chunk):
|
||||
hi=lo+edge_chunk;r=receiver[lo:hi];s=sender[lo:hi]
|
||||
weight=self.edge(attributes[lo:hi])*m[s,None]
|
||||
num.index_add_(0,r,weight*values[s]);den.index_add_(0,r,weight)
|
||||
occupied=torch.bincount(receiver,minlength=len(h))>0
|
||||
if not bool(torch.isfinite(den).all()) or bool((den[occupied]<=0).any()):raise ValueError('Invalid local denominator')
|
||||
safe=torch.where(occupied[:,None],den,torch.ones_like(den))
|
||||
return num/safe
|
||||
|
||||
def global_message(self,h,m,components,node_chunk,detach_global=False):
|
||||
out=h.new_zeros(h.shape)
|
||||
for ids in components:
|
||||
numerator=h.new_zeros((64,64));denominator=h.new_zeros((64,1));chunks=[]
|
||||
for lo in range(0,len(ids),node_chunk):
|
||||
ix=ids[lo:lo+node_chunk];hh=h[ix]
|
||||
assignment=torch.softmax(self.assignment(hh),dim=-1)
|
||||
weighted=assignment*m[ix,None]
|
||||
numerator=numerator+weighted.T@self.global_value(hh)
|
||||
denominator=denominator+weighted.sum(0)[:,None]
|
||||
chunks.append((ix,assignment))
|
||||
if not bool(torch.isfinite(denominator).all()) or bool((denominator<=0).any()):raise ValueError('Invalid token denominator')
|
||||
z=numerator/denominator
|
||||
if detach_global:z=z.detach() # Only the synthetic negative control enables this.
|
||||
z=self.attention(z[None],z[None],z[None],need_weights=False)[0][0]
|
||||
for ix,assignment in chunks:out.index_copy_(0,ix,assignment@z)
|
||||
return out
|
||||
|
||||
def forward(self,h,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global=False):
|
||||
h=h+self.local(self.local_norm(h),m,receiver,sender,attributes,edge_chunk)
|
||||
h=h+self.global_message(self.global_norm(h),m,components,node_chunk,detach_global)
|
||||
return h+self.ffn(self.ffn_norm(h))
|
||||
|
||||
class H2Geo(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__();self.encoder=nn.Sequential(nn.Linear(19,64),nn.GELU(),nn.Linear(64,64))
|
||||
self.blocks=nn.ModuleList([GeoBlock() for _ in range(4)])
|
||||
self.head=nn.Sequential(nn.Linear(83,64),nn.GELU(),nn.Linear(64,1))
|
||||
nn.init.zeros_(self.head[-1].weight);nn.init.zeros_(self.head[-1].bias)
|
||||
|
||||
def forward(self,x,m,receiver,sender,attributes,component_ids,*,edge_chunk=EDGE_CHUNK,node_chunk=NODE_CHUNK,checkpoint_blocks=True,detach_global=False):
|
||||
if x.dtype!=torch.float32 or m.dtype!=torch.float32 or attributes.dtype!=torch.float32:raise ValueError('Only FP32 model/aggregations')
|
||||
if not bool(torch.isfinite(m).all()) or bool((m<=0).any()):raise ValueError('Nonpositive mass; no epsilon')
|
||||
if bool((component_ids[receiver]!=component_ids[sender]).any()):raise ValueError('Cross-component edge')
|
||||
components=[torch.nonzero(component_ids==c,as_tuple=True)[0] for c in torch.unique(component_ids)]
|
||||
h=self.encoder(x)
|
||||
for block in self.blocks:
|
||||
def run(v,b=block):return b(v,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global)
|
||||
h=checkpoint(run,h,use_reentrant=False,preserve_rng_state=False) if checkpoint_blocks and self.training and torch.is_grad_enabled() else run(h)
|
||||
outputs=[]
|
||||
for lo in range(0,len(x),node_chunk):outputs.append(self.head(torch.cat([h[lo:lo+node_chunk],x[lo:lo+node_chunk]],dim=-1))[:,0])
|
||||
return torch.cat(outputs)
|
||||
|
||||
def optimizer(model):
|
||||
decay=[];no_decay=[]
|
||||
for name,p in model.named_parameters():
|
||||
(no_decay if name.endswith('bias') or p.ndim==1 else decay).append(p)
|
||||
return torch.optim.AdamW([dict(params=decay,weight_decay=1e-4),dict(params=no_decay,weight_decay=0.)],lr=lr_epoch(1),betas=(.9,.999),eps=1e-8)
|
||||
|
||||
def weighted_loss(prediction,target,parts,part_nodes,total_nodes):
|
||||
if len(prediction)!=part_nodes or prediction.shape!=target.shape:raise ValueError('Full part loss shape')
|
||||
return torch.mean((prediction-target)**2)*(parts*part_nodes/total_nodes)
|
||||
|
||||
def comparison(a,b):
|
||||
a=a.detach();b=b.detach();difference=float((a-b).abs().max());scale=float(a.abs().max())
|
||||
return dict(max_absolute=difference,reference_scale=scale,relative_to_scale=difference/max(scale,1e-30),passed=difference<=ATOL+RTOL*scale)
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing."""
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection
|
||||
from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_residual_math import decode_seconds
|
||||
|
||||
def peaks():
|
||||
return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
|
||||
def seed():
|
||||
random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
|
||||
def train_step(model,opt,data,parts,total):
|
||||
model.train();opt.zero_grad(set_to_none=True)
|
||||
prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total)
|
||||
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
|
||||
loss.backward()
|
||||
norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
|
||||
opt.step()
|
||||
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
|
||||
torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks())
|
||||
|
||||
def run(root):
|
||||
c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0
|
||||
(out/'history').mkdir()
|
||||
def record(value):
|
||||
nonlocal history_index
|
||||
item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value)
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'history'/f'{history_index:05d}.json',item);history_index+=1
|
||||
save(out/'progress.json',item);return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs')
|
||||
stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
props=torch.cuda.get_device_properties(0)
|
||||
save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0))))
|
||||
fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD'
|
||||
for case in c['representatives']:
|
||||
seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize()
|
||||
for step in range(2):
|
||||
tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r))
|
||||
records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks()))
|
||||
del data,opt,model;gc.collect();torch.cuda.empty_cache()
|
||||
save(out/'representative-timings.json',records)
|
||||
stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[]
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total)
|
||||
trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data
|
||||
torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start
|
||||
if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch')
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(2)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt')
|
||||
save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt')))
|
||||
stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize()
|
||||
validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds
|
||||
duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start
|
||||
projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected)
|
||||
save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Labelled S1 amendment; original pilot and scientific sources remain immutable."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked
|
||||
from phase2_h2_data import config as pilot_config
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
||||
old=pilot_config(Path(c['pilot_root']))
|
||||
if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed')
|
||||
if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap')
|
||||
return c,old
|
||||
|
||||
def gate(field_high,last_high):
|
||||
return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0))
|
||||
|
||||
def selected_checkpoint(completed_epochs):
|
||||
if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated')
|
||||
return 'last-epoch200.pt'
|
||||
|
||||
def stats_from_pilot(c,old):
|
||||
stats=read(checked(c['standardization_path'],c['standardization_sha256']))
|
||||
if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch')
|
||||
if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization')
|
||||
return stats
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
"""Local immutable barriers for explicitly approved /home run roots."""
|
||||
import fcntl,json,shutil,subprocess,hashlib
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import sha,checked,save
|
||||
from phase2_local_freeze_io import STORE,LOCK,BRANCH,BASE
|
||||
HOME_RUNS=Path('/home/frank14f/MeltMind-runs')
|
||||
def key(path):
|
||||
p=Path(path).resolve()
|
||||
if p.is_relative_to(HOME_RUNS):return 'home-runs/'+str(p.relative_to(HOME_RUNS))
|
||||
return str(p.relative_to(BASE))
|
||||
def freeze(root,name,files):
|
||||
with LOCK.open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
def git(*a):return subprocess.check_output(['git','-C',str(STORE),'-c','gc.auto=0',*a],text=True).strip()
|
||||
if git('branch','--show-current')!=BRANCH or git('status','--porcelain'):raise RuntimeError('Local writer not clean')
|
||||
hashes={};paths={}
|
||||
for f in files:
|
||||
f=Path(f).resolve();k=key(f);p=STORE/k;p.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(f,p);hashes[str(f)]=sha(f);paths[str(f)]=k
|
||||
subprocess.run(['git','-C',str(STORE),'add','--',*paths.values()],check=True)
|
||||
if git('diff','--cached','--name-only'):subprocess.run(['git','-C',str(STORE),'-c','user.name=MeltAstra','-c','user.email=meltrastra@local','commit','-q','-m','Immutable home run '+root.name+' '+name],check=True)
|
||||
commit=git('rev-parse','HEAD')
|
||||
for p,h in hashes.items():
|
||||
if hashlib.sha256(subprocess.check_output(['git','-C',str(STORE),'show',commit+':'+paths[p]])).hexdigest()!=h:raise ValueError('Git content mismatch')
|
||||
save(root/(name+'-git-freeze.json'),dict(commit=commit,files=hashes,git_paths=paths,remote_published=False,local_repository=str(STORE)))
|
||||
save(root/(name+'-authorization.json'),dict(contract_sha256=sha(root/'contract.json'),proof_sha256=sha(root/(name+'-git-freeze.json'))))
|
||||
def verify(root,name):
|
||||
a=json.loads((root/(name+'-authorization.json')).read_text());checked(root/'contract.json',a['contract_sha256']);p=checked(root/(name+'-git-freeze.json'),a['proof_sha256']);d=json.loads(p.read_text())
|
||||
for f,h in d['files'].items():checked(f,h)
|
||||
return d
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""6657 engineering-only local Git barriers; unchanged scientific proof verifier."""
|
||||
import fcntl,hashlib,json,shutil,subprocess
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import sha,save
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
STORE=Path('/tmp/meltmind-phase2-local-fixed-20261117')
|
||||
LOCK=Path('/tmp/meltmind-phase2-local-fixed-20261117.lock')
|
||||
BRANCH='codex/phase2-local-fixed-20261117'
|
||||
ENGINEERING=BASE/'artifacts/phase2-local-launch-2026-11-17/engineering-contract.json'
|
||||
|
||||
def freeze(root,name,files):
|
||||
with LOCK.open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
def git(*args):return subprocess.check_output(['git','-C',str(STORE),'-c','gc.auto=0',*args],text=True).strip()
|
||||
if not (STORE/'.git').exists():
|
||||
STORE.mkdir(exist_ok=True)
|
||||
if list(STORE.iterdir()):raise RuntimeError('Unrecognized local evidence store')
|
||||
subprocess.run(['git','init','-q','-b',BRANCH,str(STORE)],check=True)
|
||||
if git('branch','--show-current')!=BRANCH or git('status','--porcelain'):raise RuntimeError('Local barrier writer not clean')
|
||||
hashes={}
|
||||
extra=[]
|
||||
if ENGINEERING.exists():
|
||||
e=json.loads(ENGINEERING.read_text());extra=[ENGINEERING,*[BASE/p for p in e['engineering_sha256']]]
|
||||
for p,h in e['engineering_sha256'].items():
|
||||
if sha(BASE/p)!=h:raise RuntimeError('Engineering source changed: '+p)
|
||||
for p in [*files,*extra]:
|
||||
p=Path(p);rel=str(p.resolve().relative_to(BASE));dest=STORE/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest);hashes[rel]=sha(p)
|
||||
subprocess.run(['git','-C',str(STORE),'add','--',*hashes],check=True)
|
||||
if git('diff','--cached','--name-only'):
|
||||
subprocess.run(['git','-C',str(STORE),'-c','user.name=MeltAstra','-c','user.email=meltrastra@local','commit','-q','-m','Phase2 local immutable '+root.name+' '+name],check=True)
|
||||
commit=git('rev-parse','HEAD')
|
||||
for rel,h in hashes.items():
|
||||
if hashlib.sha256(subprocess.check_output(['git','-C',str(STORE),'show',commit+':'+rel])).hexdigest()!=h:raise ValueError('Local Git byte mismatch')
|
||||
save(root/(name+'-git-freeze.json'),dict(commit=commit,sha256=hashes,remote_published=False,local_repository=str(STORE),backend='6657_local_immutable_barrier',approval='1794906657.779289'))
|
||||
save(root/(name+'-authorization.json'),dict(proof_sha256=sha(root/(name+'-git-freeze.json')),contract_sha256=sha(root/'contract.json')))
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""Array-only residual interface; no I/O, model, labels, or clock fitting policy.
|
||||
|
||||
The caller supplies a frozen, training-only clock. It also fixes the F precision
|
||||
in the reviewed contract. Passing F explicitly avoids a hidden extra feature.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
|
||||
def learning_clock(end_ratios, component_ids):
|
||||
"""Mean over components of mean retained-tet-end/t_set per learning part.
|
||||
|
||||
For the random v4 protocol each part can be its own unit, if the final
|
||||
specification chooses part weighting. Caller enforces training roles.
|
||||
"""
|
||||
ratios = np.asarray(end_ratios, dtype=np.float64)
|
||||
if ratios.ndim != 1 or not len(ratios) or len(component_ids) != len(ratios):
|
||||
raise ValueError('Clock row count mismatch')
|
||||
if not np.isfinite(ratios).all() or np.any(ratios <= 0):
|
||||
raise ValueError('Invalid retained-tet endpoint ratio')
|
||||
groups = {}
|
||||
for group, ratio in zip(component_ids, ratios):
|
||||
if not isinstance(group, str) or not group:
|
||||
raise ValueError('Missing clock group identity')
|
||||
groups.setdefault(group, []).append(float(ratio))
|
||||
return float(np.mean([np.mean(v) for v in groups.values()]))
|
||||
|
||||
|
||||
def prior(cdf, clock):
|
||||
f = np.asarray(cdf)
|
||||
if f.ndim != 1 or f.dtype.kind != 'f' or not len(f):
|
||||
raise ValueError('Expected nonempty floating-point CDF vector')
|
||||
if not np.isfinite(f).all() or np.any(f < 0) or np.any(f > 1):
|
||||
raise ValueError('CDF outside [0,1] or nonfinite')
|
||||
if not np.isscalar(clock) or not np.isfinite(clock) or clock <= 0:
|
||||
raise ValueError('Clock must be a finite positive frozen scalar')
|
||||
return np.asarray(f, dtype=np.float64)*float(clock)
|
||||
|
||||
|
||||
def residual_target(target_star, cdf, clock, *, fit_dtype):
|
||||
y = np.asarray(target_star, dtype=np.float64)
|
||||
base = prior(cdf, clock)
|
||||
if y.shape != base.shape or not np.isfinite(y).all() or np.any(y < 0):
|
||||
raise ValueError('Invalid original normalized arrival target')
|
||||
dtype = np.dtype(fit_dtype)
|
||||
if dtype not in (np.dtype('float32'), np.dtype('float64')):
|
||||
raise ValueError('Fit precision must be explicitly float32 or float64')
|
||||
out = np.asarray(y-base, dtype=dtype)
|
||||
if not np.isfinite(out).all():
|
||||
raise ValueError('Residual overflow at fit precision')
|
||||
# Negative residuals are valid. Never clip or impose a positivity loss here.
|
||||
return out
|
||||
|
||||
|
||||
def decode_seconds(predicted_residual, cdf, clock, t_set):
|
||||
residual = np.asarray(predicted_residual, dtype=np.float64)
|
||||
base = prior(cdf, clock)
|
||||
if residual.shape != base.shape or not np.isfinite(residual).all():
|
||||
raise ValueError('Invalid predicted residual')
|
||||
if not np.isscalar(t_set) or not np.isfinite(t_set) or t_set <= 0:
|
||||
raise ValueError('Invalid pre-solve seconds scale')
|
||||
out = (base+residual)*float(t_set)
|
||||
if not np.isfinite(out).all():
|
||||
raise ValueError('Decoded prediction overflow')
|
||||
# Keep negative predictions visible to scoring/diagnostics; no silent clamp.
|
||||
return out
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Prepare the sole labelled recovery metadata/binding; no training or GPU work."""
|
||||
import copy,time
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import sha,checked,save
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_h2_common import read,config as old_config
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from track_a_recovery_common import OLD,ROOT,ORIGINAL,reuse_identity,project_total
|
||||
|
||||
B=Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if ROOT.exists():raise FileExistsError('Existing recovery entry; never duplicate')
|
||||
old,_=old_config(ORIGINAL);audit=read(checked(ORIGINAL/'infrastructure-stop-audit.json','d0433cb52d2695fcc5a9bb885521ff7fda2aea17524e894e7218d23a75ebe1ea'))
|
||||
verify(ORIGINAL,'infrastructure-stop');done=read(ORIGINAL/'complete.json')
|
||||
if done['status']!='STOPPED' or done['training_gpu_seconds']!=OLD:raise ValueError('Original stopped ledger')
|
||||
models=[]
|
||||
for m in audit['checkpoints']:
|
||||
i=m['job'];e=next(e for e in done['entries'] if e['id']==i)
|
||||
models.append(dict(job=i,path=m['path'],sha256=m['sha256'],standardization=m['standardization'],standardization_sha256=m['standardization_sha256'],origin='ORIGINAL_HASH_REUSE',training_contract_sha256=sha(ORIGINAL/'contract.json'),guard_complete_sha256=e['guard_complete_sha256']))
|
||||
sources={**old['source_sha256'],**old['code_sha256']}
|
||||
for name in ['contract.json','complete.json','global-stop.json','terminal-ledger.json','infrastructure-stop-audit.json','infrastructure-stop-git-freeze.json','infrastructure-stop-authorization.json','review.json']:
|
||||
sources[str(ORIGINAL/name)]=sha(ORIGINAL/name)
|
||||
for m in models:
|
||||
for key in ['path','standardization']:sources[m[key]]=sha(m[key])
|
||||
for name in ['result.json','standardization.json']:
|
||||
p=ORIGINAL/'jobs'/str(m['job'])/'worker'/name;sources[str(p)]=sha(p)
|
||||
p=ORIGINAL/'jobs'/str(m['job'])/'complete.json';sources[str(p)]=sha(p)
|
||||
post=ORIGINAL.with_name('track-a-postfit-2026-11-18')
|
||||
for name in ['contract.json','review.json','expected-review-binding.json','implementation-git-freeze.json']:
|
||||
sources[str(post/name)]=sha(post/name)
|
||||
codes=local_code_closure([B/'tools'/n for n in ['track_a_recovery_common.py','track_a_recovery_monitor.py','track_a_recovery_supervisor.py','track_a_recovery_guard.py','track_a_recovery_queue.py','track_a_recovery_worker.py','prepare_track_a_recovery.py','test_track_a_recovery.py']])
|
||||
comparison=dict(approval='1795000626.905989',monitor_ruling='1795000465.748129',original_contract_sha256=sha(ORIGINAL/'contract.json'),original_scientific_comparison_sha256=old['comparison_contract_sha256'],reused_folds=[0,1,2],fresh_seed0_folds=[3,4],epochs=200,last_epoch_only=True,interrupted_weights_reused=False,old_gpu_seconds_once=OLD,cumulative_gpu_cap=162000,new_train_cap=154800-OLD,postfit_reserve=7200,per_new_job_cap=30960,
|
||||
monitoring=dict(gpu_timeout_seconds=10,failed_samples=3,consecutive_per_operation=True,all_failures_logged=True,valid_sample_resets_own_streak=True,missing_sample_is_not_breach=True,observed_resource_breach_immediate=True,fast_local_checks_independent_of_gpu_query=True,worker_supervisor_tolerance_same_rule=True),
|
||||
isolation='Failed worker, query monitor, own projection or own idle cap affects only its fold; cumulative resource/source integrity/operator stop remains global',
|
||||
idle='Original assigned GPU2/3 only; actual idle sample then3guard idle samples; busy pending waits are charged per waiting fold inside own30960 and total budget; no interference with seed1',
|
||||
no_further_recovery=True,postfit=dict(original_review_sha256=sha(post/'review.json'),exact_binding_compatible=False,reason='Original postfit pins stopped old root and requires old five new workers. Mixed original/recovery checkpoints have two training-contract identities and tolerable logged sampling errors. Identity-only new postfit binding required before any test access; original statistics/pairing stay unchanged.',partial_cohort_scoring_authorized=False,auto_start=False))
|
||||
c=copy.deepcopy(old)
|
||||
c.update(version='track-a-labelled-infrastructure-recovery-v1',approval='1795000626.905989',monitor_ruling='1795000465.748129',original_root=str(ORIGINAL),original_contract_sha256=sha(ORIGINAL/'contract.json'),recovery_number=1,recovery_folds=[3,4],reused_folds=[0,1,2],reused_models=models,new_fits=2,no_further_recovery=True,old_gpu_seconds_once=OLD,train_gpu_cap=154800-OLD,gpu_query_timeout_seconds=10,consecutive_failed_samples=3,job_failure_isolated=True,source_sha256=sources,code_sha256=codes,postfit='No automatic execution; original postfit exact identity incompatible; explicit new identity binding must retain original statistics and complete-five gate',no_auto_launch=True)
|
||||
reuse_identity(c);assert project_total(c,{}, {},{})['status']=='PASS'
|
||||
ROOT.mkdir();save(ROOT/'comparison-contract.json',comparison)
|
||||
c.update(comparison_contract=str(ROOT/'comparison-contract.json'),comparison_contract_sha256=sha(ROOT/'comparison-contract.json'))
|
||||
save(ROOT/'contract.json',c)
|
||||
save(ROOT/'preparation-receipt.json',dict(status='PREPARED_PENDING_NARROW_DELTA_REVIEW',unix=time.time(),metadata_cpu_seconds=time.process_time()-start,original_STOP_preserved=True,reused_models_hashed=3,model_values_loaded=False,scientific_arrays_decoded=False,target_values_read=False,gpu_query_performed=False,training_started=False,postfit_started=False,code_files=len(codes),source_files=len(sources),initial_projection=project_total(c,{}, {},{})))
|
||||
print(read(ROOT/'preparation-receipt.json'))
|
||||
|
||||
if __name__=='__main__':main()
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Bounded, recorded idle observations; never signal foreign processes."""
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
|
||||
|
||||
def inventory():
|
||||
r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3)
|
||||
return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())}
|
||||
|
||||
|
||||
def snapshot(uuid):
|
||||
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
||||
rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1
|
||||
memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100
|
||||
r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
||||
occupants=[]
|
||||
for row in csv.reader(r.stdout.splitlines()):
|
||||
if not row or row[0].strip()!=uuid:continue
|
||||
pid=int(row[1]);user=None
|
||||
try:user=psutil.Process(pid).username()
|
||||
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
||||
occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip()))
|
||||
return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants,
|
||||
idle=memory<=128 and util==0 and not occupants)
|
||||
|
||||
|
||||
def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep):
|
||||
begin=clock();consecutive=0;count=0;last=None
|
||||
while True:
|
||||
try:last=sample(uuid)
|
||||
except Exception as exc:
|
||||
last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[])
|
||||
count+=1;consecutive=consecutive+1 if last['idle'] else 0
|
||||
record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin))
|
||||
if clock()-begin>limit_seconds:break
|
||||
if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
||||
remaining=limit_seconds-(clock()-begin)
|
||||
if remaining<=0:break
|
||||
sleep(min(10,remaining))
|
||||
return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
||||
|
||||
|
||||
def recorded_wait(root,name,uuid,limit_seconds):
|
||||
path=Path(root)/'idle-gates';path.mkdir(exist_ok=True)
|
||||
def record(row):
|
||||
with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n')
|
||||
result=wait_idle(uuid,limit_seconds,record)
|
||||
with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n')
|
||||
return result
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
"""Independent future-run guard; no matrix launcher or training entry point.
|
||||
|
||||
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
|
||||
peaks while the worker is alive and again after any exit, including failure.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
|
||||
|
||||
ALLOCATOR_CONF='expandable_segments:True'
|
||||
VRAM_BYTES=15*2**30
|
||||
HOST_BYTES=16*2**30
|
||||
DISK_BYTES=5*2**30
|
||||
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
|
||||
|
||||
|
||||
def peaks_in(value):
|
||||
found={}
|
||||
if isinstance(value,dict):
|
||||
for key,child in value.items():
|
||||
if key in PEAK_KEYS:
|
||||
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
|
||||
raise ValueError('Invalid recorded CUDA peak: '+key)
|
||||
found[key]=child
|
||||
nested=peaks_in(child)
|
||||
for k,v in nested.items():found[k]=max(found.get(k,0),v)
|
||||
elif isinstance(value,list):
|
||||
for child in value:
|
||||
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
|
||||
return found
|
||||
|
||||
|
||||
class PeakRecords:
|
||||
def __init__(self):
|
||||
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
|
||||
def scan(self,root):
|
||||
# Includes progress, result and failure records, plus epoch/history JSON.
|
||||
# Running maxima survive file replacement, regression and deletion.
|
||||
for path in sorted(Path(root).rglob('*.json')):
|
||||
try:
|
||||
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
|
||||
if self.cache.get(str(path))==version:continue
|
||||
raw=path.read_bytes()
|
||||
except FileNotFoundError:continue
|
||||
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
|
||||
if not peaks:continue
|
||||
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
|
||||
self.observations.append(record)
|
||||
for key,value in peaks.items():
|
||||
if key not in self.witnesses or value>self.maxima[key]:
|
||||
self.maxima[key]=value;self.witnesses[key]=record
|
||||
return max(self.maxima.values())>VRAM_BYTES
|
||||
def as_dict(self):
|
||||
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
|
||||
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
|
||||
|
||||
|
||||
def active_command_pids(command):
|
||||
result=[]
|
||||
try:children=psutil.Process(os.getpid()).children(recursive=True)
|
||||
except psutil.NoSuchProcess:return result
|
||||
for p in children:
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
||||
return result
|
||||
|
||||
|
||||
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
|
||||
root=Path(root)
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
|
||||
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
|
||||
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
|
||||
begin=time.monotonic();initial,util=device_usage(uuid)
|
||||
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
||||
tracker=PeakRecords();done=threading.Event()
|
||||
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
|
||||
def reason(value):
|
||||
if value not in monitor['reasons']:monitor['reasons'].append(value)
|
||||
def scan_records():
|
||||
try:
|
||||
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
scan_records()
|
||||
try:
|
||||
used,_=device_usage(uuid);monitor['samples']+=1
|
||||
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
||||
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
|
||||
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
||||
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
|
||||
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
|
||||
if monitor['reasons']:
|
||||
for pid in active_command_pids(command):stop_group(pid)
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
remaining=wall_limit-(time.monotonic()-begin)-2.5
|
||||
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
|
||||
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
|
||||
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
|
||||
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set();thread.join(timeout=3)
|
||||
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
|
||||
else:scan_records() # Mandatory even if the worker failed or wrote no result.
|
||||
result_path=root/name/'result.json';result=None
|
||||
if result_path.exists():
|
||||
try:result=json.loads(result_path.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
|
||||
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
|
||||
elapsed=time.monotonic()-begin
|
||||
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
|
||||
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
|
||||
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
|
||||
save(root/(name+'-guard-receipt.json'),receipt)
|
||||
return receipt,result
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""Bounded pilot training primitives; corpus runs require a frozen ready manifest."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from s2_models.network import weighted_mse
|
||||
|
||||
|
||||
def part_weights(counts, parts_in_component, component_count, learning_parts):
|
||||
c=np.asarray(counts,dtype=np.float64)
|
||||
if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts')
|
||||
if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy')
|
||||
# Uniform shuffled part steps estimate the full component-balanced objective.
|
||||
return c/c.sum()/parts_in_component/component_count*learning_parts
|
||||
|
||||
|
||||
def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False,step_info=None):
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp):
|
||||
prediction=model(base,ids,gaps,edge if model.arm=='E' else None)
|
||||
loss=weighted_mse(prediction,target,weights)
|
||||
if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss')
|
||||
scale_before=float(scaler.get_scale()) if scaler is not None else None
|
||||
if scaler is None:
|
||||
loss.backward()
|
||||
else:
|
||||
scaler.scale(loss).backward();scaler.unscale_(optimizer)
|
||||
if scaler is None and any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()):
|
||||
raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback')
|
||||
if scaler is None:optimizer.step()
|
||||
else:scaler.step(optimizer);scaler.update()
|
||||
scale_after=float(scaler.get_scale()) if scaler is not None else None
|
||||
skipped=scaler is not None and scale_after<scale_before
|
||||
if step_info is not None:
|
||||
step_info.update(step_skipped=skipped,scale_before=scale_before,scale_after=scale_after)
|
||||
if any(not torch.isfinite(p).all() for p in model.parameters()):raise FloatingPointError('Nonfinite updated parameter')
|
||||
return float(loss.detach())
|
||||
|
||||
|
||||
class SkipPolicy:
|
||||
"""Claude 1794681576: epoch 0 exempt from fraction, never from streak cap."""
|
||||
def __init__(self):self.consecutive=0;self.start_epoch()
|
||||
def start_epoch(self):self.attempted=0;self.skipped=0
|
||||
def observe(self,skipped):
|
||||
self.attempted+=1;self.skipped+=int(skipped)
|
||||
self.consecutive=self.consecutive+1 if skipped else 0
|
||||
if self.consecutive>20:raise FloatingPointError('More than 20 consecutive AMP skipped steps')
|
||||
def summary(self):
|
||||
return dict(attempted_steps=self.attempted,skipped_steps=self.skipped,successful_updates=self.attempted-self.skipped,
|
||||
skipped_fraction=self.skipped/self.attempted if self.attempted else 0.,consecutive_skipped=self.consecutive)
|
||||
def finish_epoch(self,epoch):
|
||||
if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0')
|
||||
|
||||
|
||||
def common_epoch_limit(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes):
|
||||
if min(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes)<=0:raise ValueError('Positive per-node timing and node counts required')
|
||||
epoch=training_seconds_per_node*learning_nodes+inference_seconds_per_node*validation_nodes
|
||||
if not math.isfinite(epoch):raise ValueError('Nonfinite epoch time')
|
||||
limit=min(200,math.floor(25*60/epoch))
|
||||
if limit<1:raise ValueError('One projected epoch exceeds the 25-minute run reserve')
|
||||
return dict(E_max=limit,cosine_T_max=limit,projected_epoch_seconds=epoch,projected_epochs_seconds=limit*epoch,
|
||||
training_seconds_per_node=training_seconds_per_node,inference_seconds_per_node=inference_seconds_per_node,
|
||||
learning_nodes=learning_nodes,validation_nodes=validation_nodes)
|
||||
|
||||
|
||||
def component_score(part_errors,component_ids):
|
||||
if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster')
|
||||
groups={}
|
||||
for part,value in part_errors.items():
|
||||
if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score')
|
||||
groups.setdefault(component_ids[part],[]).append(value)
|
||||
return float(np.mean([np.mean(values) for values in groups.values()]))
|
||||
|
||||
|
||||
def select_capacity(scores):
|
||||
# Exactly four run scores, no choice from an incomplete/failed run matrix.
|
||||
if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required')
|
||||
if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite')
|
||||
return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w))
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage
|
||||
from s2_pilot_train import select_capacity
|
||||
from s2_gpu_idle_gate import recorded_wait
|
||||
|
||||
|
||||
def parse_device_usage(text):
|
||||
lines=text.strip().splitlines()
|
||||
if len(lines)!=1:raise ValueError('Expected exactly one GPU')
|
||||
fields=lines[0].split(',')
|
||||
if len(fields)!=2:raise ValueError('Malformed GPU usage row')
|
||||
memory,util=map(lambda s:int(s.strip()),fields)
|
||||
if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters')
|
||||
return memory*2**20,util
|
||||
|
||||
|
||||
def device_usage(uuid):
|
||||
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2)
|
||||
return parse_device_usage(r.stdout)
|
||||
|
||||
|
||||
def active_training_pids(parent,root,index):
|
||||
ids=[]
|
||||
try:children=psutil.Process(parent).children(recursive=True)
|
||||
except psutil.NoSuchProcess:return ids
|
||||
for p in children:
|
||||
try:
|
||||
cmd=p.cmdline()
|
||||
if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return ids
|
||||
|
||||
|
||||
def guarded_worker(root,spec,i,uuid,wall_limit,command):
|
||||
run_start=time.monotonic();initial,util=device_usage(uuid)
|
||||
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
||||
done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0)
|
||||
def watch():
|
||||
previous=time.monotonic()
|
||||
while not done.is_set():
|
||||
now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now
|
||||
try:
|
||||
used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
||||
if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT'
|
||||
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
||||
if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT'
|
||||
if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT'
|
||||
except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc)
|
||||
if monitor['reason']:
|
||||
for pid in active_training_pids(os.getpid(),root,i):stop_group(pid)
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
wall=wall_limit-(time.monotonic()-run_start)-2.5
|
||||
if wall<=0:raise RuntimeError('No budget left before spawn')
|
||||
r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall,
|
||||
tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set();thread.join(timeout=3)
|
||||
total_wall=time.monotonic()-run_start
|
||||
result_path=root/spec['name']/'result.json'
|
||||
result=json.loads(result_path.read_text()) if result_path.exists() else None
|
||||
if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason'])
|
||||
if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT')
|
||||
if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT')
|
||||
r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall,
|
||||
result_sha256=sha(result_path) if result else None)
|
||||
save(root/(spec['name']+'-guard-receipt.json'),r)
|
||||
return r,result
|
||||
|
||||
|
||||
def run(root,prior_wall_seconds=0.):
|
||||
with (root/'pilot.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root,prior_wall_seconds)
|
||||
|
||||
|
||||
def run_locked(root,prior_wall_seconds=0.):
|
||||
c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid']
|
||||
prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.))
|
||||
assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)]
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
assert not (root/'pilot-started.json').exists(),'No automatic pilot retry'
|
||||
start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
rows=[];scores={};reason=None
|
||||
for i,spec in enumerate(c['runs']):
|
||||
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
|
||||
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
|
||||
# Initial readiness was proved by the continuation entry; subsequent
|
||||
# contexts also need a settling window. All waits consume the same budget.
|
||||
if i>0:
|
||||
gate=recorded_wait(root,spec['name'],uuid,min(600,remaining-3))
|
||||
if gate['status']!='IDLE_CONFIRMED':reason='GPU_IDLE_TIMEOUT_BEFORE_RUN';break
|
||||
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
|
||||
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
|
||||
command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)]
|
||||
try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command)
|
||||
except Exception as exc:reason=str(exc);break
|
||||
rows.append(dict(run=spec,**r))
|
||||
if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break
|
||||
scores[(spec['arm'],spec['width'])]=result['best_validation_score']
|
||||
if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT'
|
||||
selected=select_capacity(scores) if len(scores)==4 and reason is None else None
|
||||
summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected,
|
||||
selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required',
|
||||
pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0)
|
||||
save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def disk_usage(root):
|
||||
logical=allocated=0
|
||||
for p in Path(root).rglob('*'):
|
||||
try:s=p.stat(follow_symlinks=False)
|
||||
except FileNotFoundError:continue # atomic progress replacement / vanished entry
|
||||
if stat.S_ISREG(s.st_mode):
|
||||
logical+=s.st_size;allocated+=s.st_blocks*512
|
||||
return max(logical,allocated)
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
|
||||
def record(self,operation,exc):
|
||||
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
|
||||
try:
|
||||
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
|
||||
except Exception as log_exc:
|
||||
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
||||
def sample(self,operation,fn,previous):
|
||||
try:return fn(),True
|
||||
except Exception as exc:self.record(operation,exc);return previous,False
|
||||
def retry(self,operation,fn,poll):
|
||||
while True:
|
||||
value,ok=self.sample(operation,fn,None)
|
||||
if ok:return value
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
|
||||
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
|
||||
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
|
||||
return None
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
"""Synthetic-only recovery reuse, monitoring, isolation and ledger checks."""
|
||||
import ast,copy,json,subprocess,sys,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from track_a_recovery_monitor import Monitor,MonitorFailure,device_usage
|
||||
from track_a_recovery_common import OLD,project_total,canonical_models,accepted_guard,first_epoch_release
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
B=Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def test_two_failures_then_success_reset_and_retain_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
m=Monitor(Path(t)/'errors.jsonl')
|
||||
def bad():raise subprocess.TimeoutExpired(['nvidia-smi'],10)
|
||||
self.assertEqual(m.sample('gpu',bad),(None,False));m.sample('gpu',bad)
|
||||
self.assertFalse(m.failed);self.assertEqual(m.sample('gpu',lambda:(12,0)),((12,0),True));self.assertEqual(m.streaks['gpu'],0)
|
||||
m.sample('gpu',bad);self.assertFalse(m.failed);self.assertEqual(m.errors,3)
|
||||
self.assertEqual(len((Path(t)/'errors.jsonl').read_text().splitlines()),3)
|
||||
|
||||
def test_third_consecutive_failure_is_terminal(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
m=Monitor(Path(t)/'errors.jsonl')
|
||||
for _ in range(3):m.sample('gpu',lambda:1/0)
|
||||
self.assertTrue(m.failed);m.sample('gpu',lambda:0);self.assertTrue(m.failed)
|
||||
|
||||
def test_success_in_other_stream_does_not_hide_gpu_failures(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
m=Monitor(Path(t)/'errors.jsonl')
|
||||
for _ in range(3):m.sample('gpu',lambda:1/0);m.sample('rss',lambda:100)
|
||||
self.assertTrue(m.failed);self.assertEqual(m.max_streaks['gpu'],3)
|
||||
|
||||
def test_missing_is_not_zero_resource_sample(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
m=Monitor(Path(t)/'errors.jsonl');value,ok=m.sample('gpu',lambda:1/0)
|
||||
self.assertIsNone(value);self.assertFalse(ok);self.assertFalse(m.failed)
|
||||
|
||||
def test_final_sample_retry_bounded_at_three(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
m=Monitor(Path(t)/'errors.jsonl')
|
||||
with self.assertRaises(MonitorFailure):m.retry('final',lambda:1/0,0)
|
||||
self.assertEqual(m.errors,3)
|
||||
|
||||
def test_device_query_uses_ten_seconds_and_parses(self):
|
||||
with patch('track_a_recovery_monitor.subprocess.run',return_value=type('R',(),dict(stdout='4, 0\n'))()) as run:
|
||||
self.assertEqual(device_usage('fixed'),(4*2**20,0));self.assertEqual(run.call_args.kwargs['timeout'],10)
|
||||
|
||||
def test_original_worker_science_ast_unchanged(self):
|
||||
old=ast.parse((B/'tools/track_a_h2_worker.py').read_text());new=ast.parse((B/'tools/track_a_recovery_worker.py').read_text())
|
||||
for n in ast.walk(new):
|
||||
if isinstance(n,ast.ImportFrom) and n.module=='track_a_recovery_common':n.module='track_a_h2_common'
|
||||
run=next(n for n in new.body if isinstance(n,ast.FunctionDef) and n.name=='run')
|
||||
self.assertIsInstance(run.body[1],ast.If);run.body.pop(1)
|
||||
self.assertEqual(ast.dump(old,include_attributes=False),ast.dump(new,include_attributes=False))
|
||||
|
||||
def cfg(self):return dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200)
|
||||
|
||||
def test_old_once_and_only_two_new_jobs_in_projection(self):
|
||||
p=project_total(self.cfg(),{},{},{})
|
||||
self.assertEqual(p['new_projected_gpu_seconds'],61920+7200)
|
||||
self.assertEqual(p['cumulative_projected_gpu_seconds'],OLD+61920+7200);self.assertEqual(p['status'],'PASS')
|
||||
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {0:dict(gpu_seconds_charged=100)}, {})
|
||||
|
||||
def test_failed_terminal_cost_not_dropped_and_replaces_projection(self):
|
||||
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=30000)},{3:dict(gpu_seconds_charged=123,status='FAILED')},{4:100})
|
||||
self.assertEqual(p['new_projected_gpu_seconds'],123+30960+7200)
|
||||
|
||||
def test_actual_charge_floor_and_nonfinite_rejection(self):
|
||||
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=200)}, {},{3:500})
|
||||
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],500)
|
||||
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {},{3:float('nan')})
|
||||
|
||||
def test_no_partial_model_adoption(self):
|
||||
c=dict(reused_models=[dict(job=i) for i in range(3)])
|
||||
self.assertEqual(len(canonical_models(c,[dict(job=3),dict(job=4)])),5)
|
||||
for models in [[],[dict(job=3)],[dict(job=3),dict(job=3)]]:
|
||||
with self.assertRaises(ValueError):canonical_models(c,models)
|
||||
|
||||
def test_transient_logged_monitor_errors_can_pass_but_fatal_cannot(self):
|
||||
w=dict(status='PASS',returncode=0,monitor_errors=2,monitor_policy=dict(failed=False,errors=2))
|
||||
g=dict(status='PASS',reasons=[],monitor_errors=[{},{}],monitor_policy=dict(failed=False,errors=2),worker=w)
|
||||
self.assertTrue(accepted_guard(0,g));self.assertFalse(accepted_guard(1,g))
|
||||
bad=copy.deepcopy(g);bad['worker']['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
|
||||
bad=copy.deepcopy(g);bad['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
|
||||
bad=copy.deepcopy(g);bad['worker']['monitor_errors']=[];self.assertFalse(accepted_guard(0,bad))
|
||||
|
||||
def test_actual_collect_failure_does_not_stop_other_job(self):
|
||||
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='collect')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);p=root/'jobs/3';p.mkdir(parents=True)
|
||||
save(p/'complete.json',dict(status='STOPPED_OR_FAILED',reasons=['THREE_CONSECUTIVE_MONITOR_FAILURES'],monitor_policy=dict(failed=True)))
|
||||
class Proc:
|
||||
def poll(self):return 1
|
||||
class Log:
|
||||
def close(self):pass
|
||||
terminal={};other=object();v=dict(process=Proc(),log=Log(),wait=10);active={3:v,4:other};calls=[]
|
||||
ns=dict(root=root,read=lambda p:json.loads(p.read_text()),accepted_guard=accepted_guard,age=lambda:20,terminal=terminal,active=active,sha=sha,ledger=lambda:calls.append('ledger'))
|
||||
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_collect','exec'),ns)
|
||||
self.assertTrue(ns['collect'](3,v));self.assertIs(active[4],other);self.assertFalse((root/'global-stop.json').exists());self.assertEqual(terminal[3]['gpu_seconds_charged'],80)
|
||||
|
||||
def test_own_cost_gate_stop_does_not_block_other_fold(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'jobs/3').mkdir(parents=True);(root/'jobs/4').mkdir();(root/'contract.json').write_text('{}')
|
||||
save(root/'jobs/3/stop.json',dict(reason='own'))
|
||||
save(root/'jobs/4/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(root/'contract.json')))
|
||||
with self.assertRaises(RuntimeError):first_epoch_release(root,3)
|
||||
self.assertTrue(first_epoch_release(root,4))
|
||||
|
||||
def test_reused_fold_execution_rejected(self):
|
||||
with self.assertRaises(PermissionError):first_epoch_release(Path('/synthetic'),0)
|
||||
|
||||
def test_dispatch_busy_gpu_never_launches_or_changes_other_queue(self):
|
||||
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'jobs/4').mkdir(parents=True)
|
||||
pending=[4];active={};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
|
||||
jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]
|
||||
ns=dict(root=root,c=dict(jobs=jobs),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=lambda u:(1024**3,90),save=save,time=__import__('time'),subprocess=subprocess,sys=sys,settle_unlaunched=lambda *a:calls.append(a))
|
||||
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
|
||||
with patch.object(subprocess,'Popen') as spawn:ns['dispatch']();spawn.assert_not_called()
|
||||
self.assertEqual(pending,[4]);self.assertFalse(calls);self.assertFalse(active)
|
||||
snap=json.loads((root/'jobs/4/dispatch-idle.json').read_text());self.assertTrue(snap['valid']);self.assertEqual(snap['charged_gpu_seconds'],160)
|
||||
|
||||
def test_dispatch_three_query_failures_isolate_unlaunched_job(self):
|
||||
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'jobs/4').mkdir(parents=True);pending=[4];other=object();active={3:other};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
|
||||
def bad(u):raise subprocess.TimeoutExpired(['nvidia-smi'],10)
|
||||
def settle(i,why):calls.append((i,why));pending.remove(i)
|
||||
ns=dict(root=root,c=dict(jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=bad,save=save,time=__import__('time'),settle_unlaunched=settle)
|
||||
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
|
||||
ns['dispatch']();ns['dispatch']();self.assertFalse(calls);ns['dispatch']()
|
||||
self.assertEqual(calls,[(4,'THREE_FAILED_DISPATCH_GPU_SAMPLES')]);self.assertIs(active[3],other)
|
||||
|
||||
def test_fast_local_guard_not_blocked_on_gpu_query(self):
|
||||
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());local=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='local_watch')
|
||||
self.assertNotIn('device_usage',ast.unparse(local));self.assertIn('WHOLE_TREE_RSS_LIMIT',ast.unparse(local));self.assertIn('OUTPUT_DISK_LIMIT',ast.unparse(local));self.assertIn('RECORDED_VRAM_LIMIT',(B/'tools/track_a_recovery_guard.py').read_text())
|
||||
|
||||
def test_supervisor_real_breach_fails_without_three_monitor_errors(self):
|
||||
from track_a_recovery_supervisor import supervise
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
r=supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=1,disk_bytes=1000000,max_processes=1),poll=.01)
|
||||
self.assertNotEqual(r['status'],'PASS');self.assertIn('rss',r['stop_reason']);self.assertFalse(r['monitor_policy']['failed'])
|
||||
|
||||
def test_supervisor_transient_tree_failures_recover_and_log(self):
|
||||
import track_a_recovery_supervisor as s
|
||||
real=s.tree_usage;calls=[0]
|
||||
def sample(pid):
|
||||
calls[0]+=1
|
||||
if calls[0]<=2:raise OSError('synthetic transient')
|
||||
return real(pid)
|
||||
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=sample):
|
||||
r=s.supervise([sys.executable,'-c','import time; time.sleep(.15)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
|
||||
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],2);self.assertFalse(r['monitor_policy']['failed'])
|
||||
|
||||
def test_supervisor_three_failed_tree_samples_stop_only_child(self):
|
||||
import track_a_recovery_supervisor as s
|
||||
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=OSError('synthetic persistent')):
|
||||
r=s.supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
|
||||
self.assertNotEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],3);self.assertEqual(r['stop_reason'],'three_consecutive_monitor_failures')
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
"""Track A random-split identity and cost gates; no array or target reads."""
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
|
||||
GPU_SECONDS = 162000
|
||||
PROTECTED_GPUS = {
|
||||
'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
||||
'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
}
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def validate_roles(c, entry):
|
||||
if len(c['parts']) != 615 or len(c['jobs']) != 5:
|
||||
raise ValueError('Five original splits and 615 v4 parts required')
|
||||
if c['original_failed'] != entry['original_failed']:
|
||||
raise ValueError('Original failures changed')
|
||||
for case, original in entry['H2_inputs'].items():
|
||||
f = original['features_and_targets']
|
||||
expected = dict(case=case, output=f['output'], nodes=f['nodes'],
|
||||
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
|
||||
if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']:
|
||||
raise ValueError('Frozen native v4 input identity differs')
|
||||
for j, original in zip(c['jobs'], entry['jobs']):
|
||||
if j['id'] != original['split'] or j['split_seed'] != original['split_seed']:
|
||||
raise ValueError('Original split identity changed')
|
||||
for role in ('learning', 'test'):
|
||||
if j[role] != original[role] or len(set(j[role])) != len(j[role]):
|
||||
raise ValueError('Original membership/order changed: ' + role)
|
||||
if j['validation'] or set(j['learning']) & set(j['test']):
|
||||
raise ValueError('No new holdout or test leakage')
|
||||
clock = read(checked(j['clock_path'], j['clock_sha256']))
|
||||
if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']):
|
||||
raise ValueError('Clock learning membership differs')
|
||||
if clock['groups'] != ['part:' + x for x in clock['parts']]:
|
||||
raise ValueError('Original v4 equal-part clock groups differ')
|
||||
if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0:
|
||||
raise ValueError('Clock value differs')
|
||||
if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']):
|
||||
raise ValueError('Learning node count differs')
|
||||
if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']):
|
||||
raise ValueError('Protected GPU or epoch count')
|
||||
|
||||
|
||||
def config(root):
|
||||
root = Path(root)
|
||||
c = read(root / 'contract.json')
|
||||
verify(root, 'implementation')
|
||||
if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS
|
||||
or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200
|
||||
or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS):
|
||||
raise PermissionError('Frozen Track A scope differs')
|
||||
if c['protocol_ruling'] != '1794975093.432709':
|
||||
raise PermissionError('Actual protocol ruling differs')
|
||||
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p, h)
|
||||
if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']:
|
||||
raise ValueError('Five job budgets must partition training cap')
|
||||
entry = read(c['reuse_entry'])
|
||||
validate_roles(c, entry)
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']:
|
||||
raise ValueError('Original library versions differ')
|
||||
return c, c
|
||||
|
||||
|
||||
def view_for(c, job):
|
||||
# The shared loader rejects every case outside this split's learning role.
|
||||
return dict(c, folds=[job])
|
||||
|
||||
|
||||
def project_cell(job, observation):
|
||||
epoch = observation['epoch']
|
||||
duration = observation['max_epoch_seconds']
|
||||
charged = observation['charged']
|
||||
if (not isinstance(epoch, int) or not 1 <= epoch <= 200
|
||||
or not math.isfinite(duration) or duration <= 0
|
||||
or not math.isfinite(charged) or charged < 0):
|
||||
raise ValueError('Invalid full-epoch cost evidence')
|
||||
projected = charged + 1.25 * (200 - epoch) * duration + 120
|
||||
return dict(id=job['id'], projected_gpu_seconds=projected,
|
||||
within_cap=projected <= job['gpu_seconds'])
|
||||
|
||||
|
||||
def project_total(c, estimates, terminal, active_charges=None):
|
||||
"""Pending jobs retain full reserved caps; terminal costs replace estimates."""
|
||||
active_charges = active_charges or {}
|
||||
valid = {j['id'] for j in c['jobs']}
|
||||
if (set(estimates) | set(terminal) | set(active_charges)) - valid:
|
||||
raise ValueError('Unknown job in ledger')
|
||||
total = c['postfit_gpu_reserve']
|
||||
rows = []
|
||||
for j in c['jobs']:
|
||||
i = j['id']
|
||||
value = (terminal[i]['gpu_seconds_charged'] if i in terminal else
|
||||
max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds'])))
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise ValueError('Invalid ledger value')
|
||||
total += value
|
||||
rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds']))
|
||||
return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows)
|
||||
else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total,
|
||||
postfit_reserve=c['postfit_gpu_reserve'])
|
||||
|
||||
|
||||
def first_epoch_release(root, jobid):
|
||||
if (root / 'global-stop.json').exists():
|
||||
raise RuntimeError('Track A global stop')
|
||||
p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
r = read(p)
|
||||
if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'):
|
||||
raise RuntimeError('First-epoch cost gate rejected')
|
||||
return True
|
||||
|
||||
|
||||
def dispatchable(pending, active):
|
||||
occupied = {j['gpu_uuid'] for j in active}
|
||||
if len(occupied) != len(active) or len(active) > 2:
|
||||
raise ValueError('Active GPU collision')
|
||||
selected = []
|
||||
for j in pending:
|
||||
if len(active) + len(selected) == 2:
|
||||
break
|
||||
if j['gpu_uuid'] in PROTECTED_GPUS:
|
||||
raise ValueError('Item1 GPU reservation')
|
||||
if j['gpu_uuid'] not in occupied:
|
||||
selected.append(j)
|
||||
occupied.add(j['gpu_uuid'])
|
||||
return selected
|
||||
|
||||
|
||||
def execution_command(args):
|
||||
return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py',
|
||||
'track_a_h2_worker.py'} for a in args)
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""Build the independent Track A contract using JSON metadata only. Never launch."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from track_a_h2_common import read, validate_roles
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
ENTRY = BASE / 'artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json'
|
||||
ENTRY_SHA = '8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6'
|
||||
RULING = '1794975093.432709'
|
||||
|
||||
|
||||
def local_code_closure(paths):
|
||||
found = set()
|
||||
pending = list(paths)
|
||||
while pending:
|
||||
p = Path(pending.pop()).resolve()
|
||||
if p in found:
|
||||
continue
|
||||
found.add(p)
|
||||
for node in ast.walk(ast.parse(p.read_text())):
|
||||
names = ([node.module] if isinstance(node, ast.ImportFrom) and node.module else
|
||||
[a.name for a in node.names] if isinstance(node, ast.Import) else [])
|
||||
for name in names:
|
||||
q = BASE / 'tools' / (name.split('.')[0] + '.py')
|
||||
if q.exists():
|
||||
pending.append(q)
|
||||
return {str(p): sha(p) for p in sorted(found)}
|
||||
|
||||
|
||||
def prepare(root):
|
||||
cpu = time.process_time()
|
||||
entry = read(checked(ENTRY, ENTRY_SHA))
|
||||
sources = {str(ENTRY): ENTRY_SHA}
|
||||
for key in ('A_models_freeze', 'A_predictions_freeze', 'split_manifest',
|
||||
'preparation_manifest', 'geometry_manifest', 'source_h2_recipe'):
|
||||
ref = entry[key]
|
||||
checked(ref['path'], ref['sha256'])
|
||||
sources[ref['path']] = ref['sha256']
|
||||
pilot = read(entry['source_h2_recipe']['path'])
|
||||
h1root = BASE / 'artifacts/phase2-h1-labelled-recovery-2026-11-17'
|
||||
h1models = read(h1root / 'models-terminal.json')['models']
|
||||
sources[str(h1root / 'models-terminal.json')] = sha(h1root / 'models-terminal.json')
|
||||
parts = {}
|
||||
geometry = {}
|
||||
for case, row in entry['H2_inputs'].items():
|
||||
f = row['features_and_targets']
|
||||
parts[case] = dict(case=case, output=f['output'], nodes=f['nodes'],
|
||||
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
|
||||
geometry[case] = row['geometry']
|
||||
if geometry[case]['protocol'] != 'v4_random' or geometry[case]['nodes'] != f['nodes']:
|
||||
raise ValueError('Native v4 geometry metadata mismatch')
|
||||
jobs = []
|
||||
gpus = ['GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
||||
for original in entry['jobs']:
|
||||
i = original['split']
|
||||
model = next(m for m in h1models if m['index'] == original['existing_H1_v4_job'])
|
||||
if model['status'] != 'PASS':
|
||||
raise ValueError('Original H1 clock unavailable')
|
||||
clock = read(checked(model['clock_path'], model['clock_sha256']))
|
||||
sources[model['clock_path']] = model['clock_sha256']
|
||||
jobs.append(dict(id=i, split_seed=original['split_seed'], fold=i,
|
||||
learning=original['learning'], test=original['test'], validation=[],
|
||||
missing_train=original['missing_train'], missing_test=original['missing_test'],
|
||||
clock_path=model['clock_path'], clock_sha256=model['clock_sha256'], c=clock['c'],
|
||||
learning_nodes=sum(parts[x]['nodes'] for x in original['learning']),
|
||||
learning_parts=len(original['learning']), validation_nodes=0,
|
||||
target_steps=200 * len(original['learning']), gpu_uuid=gpus[i % 2],
|
||||
worker_cpus=[8 + 2 * (i % 2)], guard_cpus=[28 + (i % 2)],
|
||||
gpu_seconds=30960, setup_gpu_seconds=60, cpu_seconds=61920,
|
||||
vram_bytes=15 * 2**30, rss_bytes=16 * 2**30, disk_bytes=8 * 2**30))
|
||||
code = local_code_closure([*[BASE / 'tools' / ('track_a_h2_' + name + '.py')
|
||||
for name in ('common', 'worker', 'guard', 'queue', 'prepare')],
|
||||
BASE / 'tools/test_track_a_h2_entry.py'])
|
||||
# Pin original scientific functions against the already frozen H2 recipe.
|
||||
for name in ('phase2_h2_data.py', 'phase2_h2_model.py', 'phase2_h2_pilot_worker.py', 'phase2_residual_math.py'):
|
||||
p = BASE / 'tools' / name
|
||||
pin = next(h for path, h in pilot['code_sha256'].items() if Path(path).name == name)
|
||||
checked(p, pin)
|
||||
c = dict(version='track-a-h2-v1', approval='1794973932.596749', protocol_ruling=RULING,
|
||||
reuse_entry=str(ENTRY), comparison_contract=str(ENTRY), comparison_contract_sha256=ENTRY_SHA,
|
||||
new_fits=5, A_refit=False, seed=0, epochs=200, no_retry=True, new_decodes=0,
|
||||
model=pilot['model'], versions=pilot['versions'], parts=parts, geometry=geometry, jobs=jobs,
|
||||
original_failed=entry['original_failed'], source_sha256=sources, code_sha256=code,
|
||||
gpu_cap=162000, train_gpu_cap=154800, postfit_gpu_reserve=7200,
|
||||
coordinator_cpus=[30, 31], aggregate_rss_bytes=40 * 2**30, aggregate_disk_bytes=48 * 2**30,
|
||||
normalization='Per-split learning-only original 19 columns, FP64 population mean/std; separate FP64 Vf skip',
|
||||
validation='None; no checkpoint selection; no test access in this training entry',
|
||||
selection='Last epoch 200 only', clock_policy='Reuse frozen v4 H1 equal successful train-part clock',
|
||||
postfit='Separate exact Assist review required before any test access; training never starts postfit',
|
||||
metric=entry['metric'], expected_pairing_counts_provisional=[304, 311, 308, 309, 305])
|
||||
validate_roles(c, entry)
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
save(root / 'contract.json', c)
|
||||
save(root / 'preparation-receipt.json', dict(status='ENTRY_PREPARED_NOT_EXECUTED',
|
||||
contract_sha256=sha(root / 'contract.json'), code_files=len(code),
|
||||
clock_values_recomputed=False, scientific_arrays_read=False, new_fits_started=False,
|
||||
prior_A_audit_rerun=False, learning_counts=[len(j['learning']) for j in jobs],
|
||||
cpu_seconds=time.process_time() - cpu, protocol_ruling=RULING))
|
||||
print(json.dumps(read(root / 'preparation-receipt.json')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prepare(Path(sys.argv[1]).resolve())
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
"""One labelled infrastructure recovery; original data/recipe and old GPU ledger fixed."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked,sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_h2_common import read,config as original_config,view_for,project_cell
|
||||
|
||||
OLD=47943.94719455298
|
||||
ORIGINAL=Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
|
||||
ROOT=ORIGINAL.with_name('track-a-recovery-2026-11-18')
|
||||
RECOVERY=(3,4)
|
||||
|
||||
|
||||
def config(root):
|
||||
root=Path(root);c=read(root/'contract.json');verify(root,'implementation')
|
||||
if (root.resolve()!=ROOT or c['approval']!='1795000626.905989' or c['monitor_ruling']!='1795000465.748129'
|
||||
or c['recovery_folds']!=[3,4] or c['reused_folds']!=[0,1,2] or c['recovery_number']!=1
|
||||
or not c['no_retry'] or not c['no_further_recovery'] or c['new_fits']!=2 or c['seed']!=0 or c['epochs']!=200
|
||||
or c['old_gpu_seconds_once']!=OLD or c['gpu_cap']!=162000 or c['train_gpu_cap']!=154800-OLD
|
||||
or c['postfit_gpu_reserve']!=7200 or c['gpu_query_timeout_seconds']!=10
|
||||
or c['consecutive_failed_samples']!=3 or not c['job_failure_isolated']):raise PermissionError('Exact recovery scope')
|
||||
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
||||
old,_=original_config(ORIGINAL)
|
||||
if c['original_contract_sha256']!=sha(ORIGINAL/'contract.json'):raise ValueError('Old contract identity')
|
||||
for k in ['parts','geometry','jobs','model','versions','normalization','clock_policy','metric','original_failed','seed','epochs']:
|
||||
if c[k]!=old[k]:raise ValueError('Scientific recipe changed: '+k)
|
||||
d=read(ORIGINAL/'complete.json')
|
||||
if d['status']!='STOPPED' or d['training_gpu_seconds']!=OLD or sum(e['gpu_seconds_charged'] for e in d['entries'])!=OLD:
|
||||
raise ValueError('Old cost counted once')
|
||||
if c['gpu_cap']-OLD-c['postfit_gpu_reserve']!=c['train_gpu_cap']:raise ValueError('Remaining training ledger')
|
||||
return c,c
|
||||
|
||||
|
||||
def reuse_identity(c):
|
||||
d=read(ORIGINAL/'complete.json');stop=read(ORIGINAL/'global-stop.json')
|
||||
if d['status']!='STOPPED' or stop['reason']!='JOB_4_FAILED':raise ValueError('Original STOP must stay immutable')
|
||||
proof=verify(ORIGINAL,'infrastructure-stop');out=[]
|
||||
for i in [0,1,2]:
|
||||
e=next(e for e in d['entries'] if e['id']==i);w=ORIGINAL/'jobs'/str(i)/'worker'
|
||||
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']));receipt=read(w.parent/'guards/train.receipt.json')
|
||||
if e['status']!='PASS' or e['returncode']!=0 or g['status']!='PASS' or g['monitor_errors'] or g['reasons'] or g['worker']!=receipt or receipt['status']!='PASS' or receipt['returncode']!=0 or type(receipt['monitor_errors']) is not int or receipt['monitor_errors']!=0:
|
||||
raise ValueError('Only genuine old PASS may be reused')
|
||||
r=read(checked(w/'result.json',g['worker_result_sha256']));j=c['jobs'][i]
|
||||
if r['fold']!=i or r['job']!=i or r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or r['test_targets_read'] or r['test_prediction_started'] or r['validation_used_for_selection']:
|
||||
raise ValueError('Old final model roles')
|
||||
model=dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='ORIGINAL_HASH_REUSE',training_contract_sha256=c['original_contract_sha256'],guard_complete_sha256=e['guard_complete_sha256'])
|
||||
for p,h in [(model['path'],model['sha256']),(model['standardization'],model['standardization_sha256'])]:
|
||||
checked(p,h)
|
||||
if proof['files'].get(str(Path(p).resolve()))!=h:raise ValueError('Reuse omitted from immutable STOP freeze')
|
||||
stats=read(model['standardization'])
|
||||
if stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or len(stats['mean'])!=19 or len(stats['scale'])!=19:raise ValueError('Original normalization identity')
|
||||
out.append(model)
|
||||
if out!=c['reused_models']:raise ValueError('Exact reused model manifest')
|
||||
return out
|
||||
|
||||
|
||||
def execution_command(args):
|
||||
return any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py','track_a_recovery_queue.py','track_a_recovery_guard.py','track_a_recovery_worker.py'} for a in args)
|
||||
|
||||
|
||||
def first_epoch_release(root,jobid):
|
||||
if jobid not in RECOVERY:raise PermissionError('No reused fold execution')
|
||||
if (root/'global-stop.json').exists() or (root/'jobs'/str(jobid)/'stop.json').exists():raise RuntimeError('This recovery fold stopped')
|
||||
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
|
||||
if not p.exists():return False
|
||||
d=read(p)
|
||||
if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Own cost gate rejected')
|
||||
return True
|
||||
|
||||
|
||||
def accepted_guard(rc,g):
|
||||
if rc!=0 or not g or g['status']!='PASS' or g['reasons'] or g['monitor_policy']['failed']:return False
|
||||
if g['monitor_policy']['errors'] != len(g['monitor_errors']):return False
|
||||
w=g.get('worker')
|
||||
return bool(w and w['status']=='PASS' and w['returncode']==0 and type(w['monitor_errors']) is int
|
||||
and not w['monitor_policy']['failed'] and w['monitor_policy']['errors']==w['monitor_errors'])
|
||||
|
||||
|
||||
def project_total(c,estimates,terminal,live):
|
||||
if (set(estimates)|set(terminal)|set(live))-set(RECOVERY):raise ValueError('Unknown recovery fold')
|
||||
rows=[];new=c['postfit_gpu_reserve']
|
||||
for i in RECOVERY:
|
||||
value=terminal[i]['gpu_seconds_charged'] if i in terminal else max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']))
|
||||
if not math.isfinite(value) or value<0:raise ValueError('Invalid new charge')
|
||||
new+=value;rows.append(dict(id=i,projected_gpu_seconds=value))
|
||||
# Per-cell excess is isolated by own stop.json; total exhaustion stays global.
|
||||
return dict(status='PASS' if OLD+new<=c['gpu_cap'] else 'STOP_CUMULATIVE_BUDGET',rows=rows,new_projected_gpu_seconds=new,old_gpu_seconds_once=OLD,cumulative_projected_gpu_seconds=OLD+new,postfit_reserve=c['postfit_gpu_reserve'])
|
||||
|
||||
|
||||
def canonical_models(c,new_models):
|
||||
ids=[m['job'] for m in new_models]
|
||||
if len(ids)!=2 or set(ids)!={3,4}:raise ValueError('No five-fold model adoption with missing recovery fold')
|
||||
result=c['reused_models']+new_models
|
||||
if len(result)!=5 or {m['job'] for m in result}!=set(range(5)):raise ValueError('Canonical five models')
|
||||
return sorted(result,key=lambda m:m['job'])
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
"""Isolated recovery fold: 10s GPU samples, three failures; fast local guards stay live."""
|
||||
import fcntl,json,os,resource,signal,sys,threading,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from track_a_recovery_common import config
|
||||
from track_a_recovery_monitor import Monitor,device_usage
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,fold):
|
||||
contract,_=config(queue)
|
||||
if fold not in contract['recovery_folds']:raise PermissionError('Only fresh folds3/4')
|
||||
root=queue/'jobs'/str(fold);job=dict(contract['jobs'][fold]);assignment=json.loads((root/'assignment.json').read_text())
|
||||
if assignment['id']!=fold or assignment['contract_sha256']!=sha(queue/'contract.json') or assignment['gpu_uuid']!=job['gpu_uuid']:raise ValueError('Fixed assignment')
|
||||
wait=assignment['pre_dispatch_wait_seconds']
|
||||
if not 0<=wait<job['gpu_seconds']-120:raise ValueError('No remaining per-fold window')
|
||||
job['gpu_seconds']-=wait
|
||||
os.sched_setaffinity(0,job['guard_cpus'])
|
||||
with (root/'fold.lock').open('a') as lock,Path('/tmp/meltmind-track-a-'+job['gpu_uuid']+'.lock').open('a') as gpu_lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One recovery attempt per fold')
|
||||
start=time.monotonic();cpu=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
reasons=[];requested=[];done=threading.Event();tracker=PeakRecords();monitor=Monitor(root/'monitor-errors.jsonl');receipt=None;threads=[]
|
||||
command=[sys.executable,'tools/track_a_recovery_worker.py',str(queue),str(fold)]
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def evaluate_errors():
|
||||
if monitor.failed:reason('THREE_CONSECUTIVE_MONITOR_FAILURES')
|
||||
if reasons:kill()
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree_usage',lambda:tree_usage(os.getpid()))
|
||||
if ok and usage[1]>job['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
disk,ok=monitor.sample('disk_usage',lambda:disk_usage(root))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=job['gpu_seconds']-15:reason('TRACK_A_FOLD_TIME_LIMIT')
|
||||
if (root/'stop.json').exists():reason('OWN_FOLD_STOP')
|
||||
if (queue/'global-stop.json').exists():reason('CUMULATIVE_OR_OPERATOR_STOP')
|
||||
for s in requested:reason(s)
|
||||
monitor.sample('local_record',lambda:save(root/'live-progress.json',dict(unix=time.time(),elapsed_gpu_seconds=elapsed(),pre_dispatch_wait_seconds=wait,recorded_peaks=tracker.maxima,reasons=list(reasons),monitor_policy=monitor.policy())))
|
||||
evaluate_errors();done.wait(.5)
|
||||
def gpu_watch():
|
||||
while not done.is_set():
|
||||
begin=time.time();value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid']))
|
||||
if ok and value[0]>job['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
monitor.sample('gpu_record',lambda:save(root/'gpu-sample.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=value[0] if ok else None,utilization=value[1] if ok else None,monitor_policy=monitor.policy())))
|
||||
evaluate_errors();done.wait(.5)
|
||||
handlers={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=job['gpu_uuid'],pre_dispatch_wait_seconds=wait,remaining_gpu_seconds=job['gpu_seconds'],fresh_seed=0))
|
||||
try:
|
||||
samples=[];success=0
|
||||
while success<3:
|
||||
if requested or (root/'stop.json').exists() or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_START')
|
||||
value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid']))
|
||||
if not ok:
|
||||
success=0
|
||||
if monitor.failed:raise RuntimeError('THREE_CONSECUTIVE_IDLE_SAMPLE_FAILURES')
|
||||
time.sleep(.5);continue
|
||||
samples.append(dict(unix=time.time(),memory_bytes=value[0],utilization=value[1]))
|
||||
save(root/'idle-polls.json',samples)
|
||||
if value[0]>128*2**20 or value[1]!=0:raise RuntimeError('SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED')
|
||||
success+=1
|
||||
if success<3:time.sleep(2)
|
||||
for fn in [local_watch,gpu_watch]:
|
||||
t=threading.Thread(target=fn,daemon=True);t.start();threads.append(t)
|
||||
remaining=job['gpu_seconds']-elapsed()-20
|
||||
if remaining<=0:raise RuntimeError('SETUP_EXHAUSTED_WINDOW')
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('recorded_peaks',scan,.5)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_FAILED: '+str(exc))
|
||||
for s,h in handlers.items():signal.signal(s,h)
|
||||
evaluate_errors()
|
||||
result=None;p=root/'worker/result.json'
|
||||
if p.exists():
|
||||
try:result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_FAILED: '+str(exc))
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if receipt and (type(receipt['monitor_errors']) is not int or receipt['monitor_policy']['failed']):reason('WORKER_MONITOR_FAILED')
|
||||
if not result or result.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
elif result['job']!=fold or result['fold']!=fold or result['completed_epochs']!=200 or result['optimizer_steps']!=job['target_steps']:reason('INCOMPLETE_FINAL_IDENTITY')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_RECORDED_PEAKS')
|
||||
if any(v>job['vram_bytes'] for v in tracker.maxima.values()):reason('RECORDED_VRAM_LIMIT')
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);childcpu=u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime
|
||||
totalcpu=60+time.process_time()-cpu+max(childcpu,receipt['cpu_seconds'] if receipt else 0)
|
||||
if elapsed()>job['gpu_seconds'] or totalcpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
monitor_errors=monitor.records,monitor_policy=monitor.policy(),worker=receipt,recorded_memory=tracker.as_dict(),
|
||||
gpu_seconds_charged=elapsed(),pre_dispatch_wait_seconds=wait,cpu_seconds_charged=totalcpu,
|
||||
child_cpu_seconds_observed=childcpu,worker_result_sha256=sha(p) if p.exists() else None,
|
||||
no_retry=True,no_further_recovery=True,prediction_started=False,scoring_started=False))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""Recovery-only consecutive failed-sample tolerance; missing is never a breach."""
|
||||
import json,subprocess,threading,time
|
||||
from pathlib import Path
|
||||
from s2_precision_continuation_guard import parse_device_usage
|
||||
|
||||
|
||||
class MonitorFailure(RuntimeError):pass
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(self,path):
|
||||
self.path=Path(path);self.errors=0;self.last_error=None;self.streaks={};self.max_streaks={};self.failed=False;self.records=[];self.lock=threading.Lock()
|
||||
|
||||
def sample(self,operation,fn,previous=None):
|
||||
try:value=fn()
|
||||
except Exception as exc:
|
||||
with self.lock:
|
||||
self.errors+=1;streak=self.streaks.get(operation,0)+1;self.streaks[operation]=streak
|
||||
self.max_streaks[operation]=max(streak,self.max_streaks.get(operation,0));self.failed|=streak>=3
|
||||
row=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc),consecutive_failures=streak,failed=streak>=3)
|
||||
self.last_error=row;self.records.append(row)
|
||||
# A logging failure is not silently discarded: stderr retains it and
|
||||
# the caller stops because failure evidence cannot be persisted.
|
||||
try:
|
||||
with self.path.open('a') as f:f.write(json.dumps(row)+'\n')
|
||||
except Exception as log_exc:
|
||||
import sys
|
||||
print(json.dumps(dict(monitor_error=row,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
||||
self.failed=True
|
||||
return previous,False
|
||||
with self.lock:self.streaks[operation]=0
|
||||
return value,True
|
||||
|
||||
def retry(self,operation,fn,poll):
|
||||
while True:
|
||||
value,ok=self.sample(operation,fn)
|
||||
if ok:return value
|
||||
if self.failed:raise MonitorFailure('Three consecutive failed samples: '+operation)
|
||||
time.sleep(poll)
|
||||
|
||||
def policy(self):
|
||||
return dict(rule='three_consecutive_failed_samples_per_operation',failure_limit=3,failed=self.failed,
|
||||
errors=self.errors,streaks=dict(self.streaks),max_streaks=dict(self.max_streaks))
|
||||
|
||||
|
||||
def device_usage(uuid):
|
||||
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=10)
|
||||
return parse_device_usage(r.stdout)
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
"""Only folds3/4 fresh; failed folds isolated, all old costs retained, never postfit."""
|
||||
import fcntl,os,resource,signal,subprocess,sys,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha,checked
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from track_a_recovery_common import config,read,reuse_identity,execution_command,accepted_guard,project_cell,project_total,canonical_models,OLD
|
||||
from track_a_recovery_monitor import Monitor,device_usage
|
||||
from track_a_recovery_supervisor import tree_usage,disk_usage
|
||||
|
||||
|
||||
def run(root):
|
||||
started_clock=time.monotonic();cpu=time.process_time();children0=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
c,_=config(root);validate_review(root);models=reuse_identity(c)
|
||||
lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One labelled recovery only')
|
||||
if any(p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []) for p in psutil.process_iter(['pid','cmdline'])):raise RuntimeError('Existing Track A queue')
|
||||
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True'
|
||||
active={};terminal={};estimates={};last_epoch={};pending=list(c['recovery_folds']);requests=[];idle={}
|
||||
for i in pending:
|
||||
(root/'jobs'/str(i)).mkdir(parents=True,exist_ok=False)
|
||||
idle[i]=Monitor(root/'jobs'/str(i)/'dispatch-monitor-errors.jsonl')
|
||||
def age():return time.monotonic()-started_clock
|
||||
def live():return {i:60+age() for i in pending+list(active)}
|
||||
def charged():return sum(e['gpu_seconds_charged'] for e in terminal.values())+sum(live().values())
|
||||
def global_stop(why):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=why,unix=time.time(),scope='cumulative_resource_integrity_or_operator_only'))
|
||||
def own_stop(i,why):
|
||||
p=root/'jobs'/str(i)/'stop.json'
|
||||
if not p.exists():save(p,dict(id=i,reason=why,unix=time.time(),other_fold_must_continue=True))
|
||||
def terminate(v):
|
||||
if v.get('stop_sent') is None:
|
||||
if v['process'].poll() is None:v['process'].send_signal(signal.SIGTERM)
|
||||
v['stop_sent']=time.monotonic()
|
||||
elif time.monotonic()-v['stop_sent']>30:
|
||||
try:
|
||||
p=psutil.Process(v['process'].pid)
|
||||
for q in reversed(p.children(recursive=True)):
|
||||
try:q.kill()
|
||||
except psutil.NoSuchProcess:pass
|
||||
p.kill()
|
||||
except psutil.NoSuchProcess:pass
|
||||
def ledger():
|
||||
save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_settled_gpu_seconds=sum(e['gpu_seconds_charged'] for e in terminal.values()),new_charged_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged()))
|
||||
def settle_unlaunched(i,reason):
|
||||
terminal[i]=dict(id=i,status='STOPPED_BEFORE_DISPATCH',returncode=None,gpu_seconds_charged=60+age(),reason=reason,monitor_policy=idle[i].policy(),no_retry=True)
|
||||
pending.remove(i);own_stop(i,reason);ledger()
|
||||
def collect(i,v):
|
||||
rc=v['process'].poll()
|
||||
if rc is None:return False
|
||||
p=root/'jobs'/str(i)/'complete.json';g=None;error=None
|
||||
try:g=read(p) if p.exists() else None;ok=accepted_guard(rc,g)
|
||||
except Exception as exc:ok=False;error=str(exc)
|
||||
cost=max(60+age(),v['wait']+(g.get('gpu_seconds_charged',0) if isinstance(g,dict) else 0))
|
||||
terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,guard_complete_sha256=sha(p) if p.exists() else None,pre_dispatch_wait_seconds=v['wait'],receipt_error=error)
|
||||
v['log'].close();del active[i];ledger()
|
||||
# Deliberately no global-stop on failed/malformed job receipt.
|
||||
return True
|
||||
def dispatch():
|
||||
for i in list(pending):
|
||||
j=c['jobs'][i]
|
||||
if 60+age()>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue
|
||||
begin=time.time();usage,ok=idle[i].sample('gpu_usage',lambda:device_usage(j['gpu_uuid']))
|
||||
save(root/'jobs'/str(i)/'dispatch-idle.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=usage[0] if ok else None,utilization=usage[1] if ok else None,monitor_policy=idle[i].policy(),charged_gpu_seconds=60+age()))
|
||||
if idle[i].failed:settle_unlaunched(i,'THREE_FAILED_DISPATCH_GPU_SAMPLES');continue
|
||||
if not ok or usage[0]>128*2**20 or usage[1]!=0:continue
|
||||
wait=age()
|
||||
if wait>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue
|
||||
p=root/'jobs'/str(i);save(p/'assignment.json',dict(id=i,gpu_uuid=j['gpu_uuid'],contract_sha256=sha(root/'contract.json'),pre_dispatch_wait_seconds=wait,unix=time.time()))
|
||||
log=(p/'entry.log').open('x')
|
||||
try:proc=subprocess.Popen([sys.executable,'tools/track_a_recovery_guard.py',str(root),str(i)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
except Exception as exc:log.close();settle_unlaunched(i,'SPAWN_FAILED: '+str(exc));continue
|
||||
active[i]=dict(process=proc,log=log,wait=wait,stop_sent=None);pending.remove(i)
|
||||
save(p/'dispatch.json',dict(pid=proc.pid,gpu_uuid=j['gpu_uuid'],unix=time.time(),pre_dispatch_wait_seconds=wait))
|
||||
handlers={s:signal.signal(s,lambda n,f:requests.append('OPERATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_fits=2,reused_folds=[0,1,2],fresh_folds=[3,4],old_gpu_seconds_once=OLD,new_train_cap=c['train_gpu_cap'],cumulative_gpu_cap=162000,test_targets_read=False,postfit_started=False))
|
||||
try:
|
||||
save(root/'reused-models.json',dict(models=models,old_gpu_seconds_once=OLD))
|
||||
freeze(root,'reuse',[root/'contract.json',root/'reused-models.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]])
|
||||
while pending or active:
|
||||
for i,v in list(active.items()):collect(i,v)
|
||||
for reason in requests:global_stop(reason)
|
||||
for p,h in c['code_sha256'].items():checked(p,h)
|
||||
if charged()>=c['train_gpu_cap']-80:global_stop('CUMULATIVE_TRAIN_BOUNDARY')
|
||||
_,rss,_=tree_usage(os.getpid())
|
||||
if rss>c['aggregate_rss_bytes'] or disk_usage(root)>c['aggregate_disk_bytes']:global_stop('AGGREGATE_RESOURCE_LIMIT')
|
||||
for i in list(active):
|
||||
rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))]
|
||||
if rows and rows[-1]['epoch']>last_epoch.get(i,0):
|
||||
epoch=rows[-1]['epoch'];cell=project_cell(c['jobs'][i],dict(epoch=epoch,max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),charged=60+age()))
|
||||
estimates[i]=cell;total=project_total(c,estimates,terminal,live())
|
||||
result=dict(status='PASS' if cell['within_cap'] and total['status']=='PASS' else 'STOP_OWN_COST_PROJECTION',cell=cell,total=total,contract_sha256=sha(root/'contract.json'),unix=time.time())
|
||||
name='first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
||||
save(root/'jobs'/str(i)/name,result);save(root/'latest-cost-projection.json',total);last_epoch[i]=epoch
|
||||
if not cell['within_cap']:own_stop(i,'OWN_PROJECTED_CAP')
|
||||
if total['status']!='PASS':global_stop('CUMULATIVE_PROJECTED_CAP')
|
||||
if not (root/'global-stop.json').exists():dispatch()
|
||||
else:
|
||||
for i in list(pending):settle_unlaunched(i,'GLOBAL_STOP_BEFORE_DISPATCH')
|
||||
for i,v in active.items():
|
||||
if (root/'global-stop.json').exists() or (root/'jobs'/str(i)/'stop.json').exists():terminate(v)
|
||||
save(root/'live-progress.json',dict(unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],pending=list(pending),terminal=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),postfit_started=False))
|
||||
if active or pending:time.sleep(1)
|
||||
config(root);reuse_identity(c);new=[]
|
||||
for i in c['recovery_folds']:
|
||||
if terminal[i]['status']!='PASS':continue
|
||||
w=root/'jobs'/str(i)/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256']));j=c['jobs'][i]
|
||||
if r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or r['test_targets_read']:raise ValueError('Recovery final identity')
|
||||
checked(r['checkpoint'],r['checkpoint_sha256'])
|
||||
new.append(dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='RECOVERY_FRESH_SEED0',training_contract_sha256=sha(root/'contract.json'),guard_complete_sha256=terminal[i]['guard_complete_sha256']))
|
||||
complete=len(new)==2 and not (root/'global-stop.json').exists() and charged()<=c['train_gpu_cap']
|
||||
available=canonical_models(c,new) if complete else sorted(models+new,key=lambda m:m['job'])
|
||||
name='models-terminal.json' if complete else 'available-models-incomplete.json'
|
||||
save(root/name,dict(models=available,new_fits=2,reused_folds=[0,1,2],all_five_complete=complete,postfit_started=False))
|
||||
freeze(root,'models' if complete else 'incomplete-models',[root/'contract.json',root/name,root/'terminal-ledger.json',*[Path(m['path']) for m in available],*[Path(m['standardization']) for m in available]])
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_REBIND_REQUIRED' if complete else 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY',entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),available_folds=[m['job'] for m in available],coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,test_targets_read=False,no_further_recovery=True,original_postfit_exact_binding_compatible=False))
|
||||
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/name,root/'terminal-ledger.json'])
|
||||
return 0 if complete else 1
|
||||
except BaseException as exc:
|
||||
global_stop(type(exc).__name__+': '+str(exc))
|
||||
for i in list(pending):settle_unlaunched(i,'COORDINATOR_ABORT')
|
||||
while active:
|
||||
for i,v in list(active.items()):
|
||||
if not collect(i,v):terminate(v)
|
||||
if active:time.sleep(.5)
|
||||
save(root/'complete.json',dict(status='RECOVERY_COORDINATOR_STOPPED',reason=str(exc),entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_further_recovery=True))
|
||||
freeze(root,'stopped',[root/'contract.json',root/'complete.json',root/'global-stop.json',root/'terminal-ledger.json'])
|
||||
raise
|
||||
finally:
|
||||
for s,h in handlers.items():signal.signal(s,h)
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if monitor.failed:
|
||||
reason = 'three_consecutive_monitor_failures'
|
||||
elif now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,monitor_policy=monitor.policy())
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
def write_receipt():
|
||||
receipt.update(monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,monitor_policy=monitor.policy())
|
||||
save(receipt_path,receipt)
|
||||
monitor.retry('receipt_write',write_receipt,poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"""Track A v4 random split; same H2 seed0/200epochs, learning-only, no test access."""
|
||||
import json,os,sys,time,traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from track_a_recovery_common import config,view_for,first_epoch_release
|
||||
from phase2_h2_data import standardization
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import seed,peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id):
|
||||
c,_=config(root)
|
||||
if fold_id not in c['recovery_folds']:raise PermissionError('Only fresh recovery folds3/4')
|
||||
job=c['jobs'][fold_id];old=view_for(c,job);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_OWN_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',job=fold_id,fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"reproductions": {
|
||||
"own_projection_escalates_to_global": {
|
||||
"own_stops": [
|
||||
[
|
||||
3,
|
||||
"OWN_PROJECTED_CAP"
|
||||
]
|
||||
],
|
||||
"global_stops": [
|
||||
"CUMULATIVE_PROJECTED_CAP"
|
||||
],
|
||||
"rejected_fold_projection": 249030.0,
|
||||
"global_projection": 335133.947194553,
|
||||
"actual_failed_fold_plus_survivor_full_reserve_and_postfit": 86263.94719455298,
|
||||
"explanation": "Known locally rejected future work still enters total and globally stops the other fold"
|
||||
},
|
||||
"prestart_idle_loop_has_no_deadline": {
|
||||
"already_expired_elapsed": 30961,
|
||||
"remaining_cap": 30960,
|
||||
"query_calls_after_expiry": 14,
|
||||
"elapsed_checks": 0,
|
||||
"max_consecutive_errors": {
|
||||
"gpu_usage": 2
|
||||
},
|
||||
"monitor_failed": false,
|
||||
"synthetic_termination": "SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED",
|
||||
"explanation": "Repeating two failures then one valid idle sample resets monitor streak but never gets three idle successes; no elapsed check bounds this loop"
|
||||
}
|
||||
},
|
||||
"real_gpu_queries": false,
|
||||
"science_executed": false,
|
||||
"source_hashes": {
|
||||
"tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
{
|
||||
"unix": 1795001943.655267,
|
||||
"old_implementation": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"old_binding_sha256": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"blocked_review_sha256": "2bd485043af0ebb42aab3bd93da0eedddcb706f652a2868280f011ee486c6599",
|
||||
"preserved_files": [
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/binding-audit.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/binding-audit.json",
|
||||
"sha256": "5a327e78162bfd94e985c0f132e1aa169b99c1b1f74088ac8ff11a00835df449",
|
||||
"bytes": 420
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/comparison-contract.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/comparison-contract.json",
|
||||
"sha256": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"bytes": 1960
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"bytes": 1379675
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/expected-review-binding.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/expected-review-binding.json",
|
||||
"sha256": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"bytes": 4073
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/implementation-authorization.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-authorization.json",
|
||||
"sha256": "a3fa7bfc860754f469f967372f0e2100cb9bde29e7ce2b45efe19ec07b89ea16",
|
||||
"bytes": 178
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/implementation-git-freeze.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-git-freeze.json",
|
||||
"sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"bytes": 11455
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/preparation-receipt.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/preparation-receipt.json",
|
||||
"sha256": "7002cfc60511d80f6aae331d255fad501eeaaa32b63f1d7aa29b36e5b2558fae",
|
||||
"bytes": 814
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json",
|
||||
"sha256": "a6c9eb98ca7d3f5e95e0b548aafcea3d980477ffba11ebb19c4809bc7d1ba386",
|
||||
"bytes": 2456
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py",
|
||||
"sha256": "a01d403f400b4adabfca32c3291dfd6228878634b963353bccd274a4924da696",
|
||||
"bytes": 3986
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-recovery-review-2026-11-18/review.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/review.json",
|
||||
"sha256": "2bd485043af0ebb42aab3bd93da0eedddcb706f652a2868280f011ee486c6599",
|
||||
"bytes": 8549
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log",
|
||||
"sha256": "85483476d5bfa7274b5a3d658fe5e57227d26063b56a6827f6a9fb5960715b4d",
|
||||
"bytes": 2659
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log",
|
||||
"sha256": "7db6ca25fbc9586e893065728b955c72877f5f16dc43e455f1b83eeb9a576fdf",
|
||||
"bytes": 2233
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json",
|
||||
"sha256": "b5e937c870225059ec38fbefbb2b933ef46cea10fb38fe520156749da37cf3c8",
|
||||
"bytes": 1573
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock",
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"bytes": 0
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json",
|
||||
"sha256": "b967bd9e67c37f99af42b8160941dc5899a5193cc15bda243e2eaa79c5100434",
|
||||
"bytes": 165
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log",
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"bytes": 0
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl",
|
||||
"sha256": "805226f37c52f96a095566dd6ee5f1e4c20871ebada2db99af679c6d197458da",
|
||||
"bytes": 306
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json",
|
||||
"sha256": "c8aa937dec2e2ca9dfdaa102c0a61e4cb11b513d7ded085887b471e6d0a2478d",
|
||||
"bytes": 1265
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json",
|
||||
"sha256": "4027923ffc3bd93b1b9cea875cbba236b56249e55a41af354cf19f226af6c441",
|
||||
"bytes": 396
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/docs/track-a-recovery-entry-2026-11-18.md",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/docs/track-a-recovery-entry-2026-11-18.md",
|
||||
"sha256": "070536c0addeeef2d2ce70edb6387f26307fad5fc1cd0df22fec0def36398732",
|
||||
"bytes": 6271
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py",
|
||||
"sha256": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"bytes": 18855
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/gre24_models.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/gre24_models.py",
|
||||
"sha256": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"bytes": 1819
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py",
|
||||
"sha256": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"bytes": 1401
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py",
|
||||
"sha256": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"bytes": 10840
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py",
|
||||
"sha256": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"bytes": 6770
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py",
|
||||
"sha256": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"bytes": 6291
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py",
|
||||
"sha256": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"bytes": 7114
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py",
|
||||
"sha256": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"bytes": 1746
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py",
|
||||
"sha256": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"bytes": 2021
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py",
|
||||
"sha256": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"bytes": 2343
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py",
|
||||
"sha256": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"bytes": 3009
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/prepare_track_a_recovery.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/prepare_track_a_recovery.py",
|
||||
"sha256": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"bytes": 5677
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py",
|
||||
"sha256": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"bytes": 2764
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py",
|
||||
"sha256": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"bytes": 6656
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py",
|
||||
"sha256": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"bytes": 4791
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py",
|
||||
"sha256": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"bytes": 6900
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py",
|
||||
"sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"bytes": 8021
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py",
|
||||
"sha256": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"bytes": 1590
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/test_track_a_recovery.py",
|
||||
"sha256": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"bytes": 12296
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py",
|
||||
"sha256": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"bytes": 6454
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py",
|
||||
"sha256": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"bytes": 6049
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_common.py",
|
||||
"sha256": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"bytes": 6517
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_guard.py",
|
||||
"sha256": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"bytes": 8161
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_monitor.py",
|
||||
"sha256": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"bytes": 2355
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_queue.py",
|
||||
"sha256": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"bytes": 11419
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_supervisor.py",
|
||||
"sha256": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"bytes": 8377
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_worker.py",
|
||||
"sha256": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5",
|
||||
"bytes": 5381
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
test_actual_charge_floor_and_nonfinite_rejection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_actual_collect_failure_does_not_stop_other_job (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_already_expired_prestart_makes_zero_queries (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_alternating_two_errors_one_idle_expires_without_worker (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_device_query_uses_ten_seconds_and_parses (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_dispatch_busy_gpu_never_launches_or_changes_other_queue (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_dispatch_three_query_failures_isolate_unlaunched_job (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_extreme_rejected_projection_cannot_stop_affordable_survivor (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_failed_terminal_cost_not_dropped_and_replaces_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_fast_local_guard_not_blocked_on_gpu_query (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_final_sample_retry_bounded_at_three (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_missing_is_not_zero_resource_sample (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_no_partial_model_adoption (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_old_once_and_only_two_new_jobs_in_projection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_original_worker_science_ast_unchanged (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_own_cost_gate_stop_does_not_block_other_fold (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_prestart_checks_after_query_overrun (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_prestart_checks_before_idle_sleep (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_reused_fold_execution_rejected (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_stopping_reserve_keeps_actual_and_terminal_settles_once (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_success_in_other_stream_does_not_hide_gpu_failures (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_real_breach_fails_without_three_monitor_errors (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_three_failed_tree_samples_stop_only_child (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_supervisor_transient_tree_failures_recover_and_log (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_third_consecutive_failure_is_terminal (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_transient_logged_monitor_errors_can_pass_but_fatal_cannot (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_true_cumulative_overrun_still_stops_after_local_rejection (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
test_two_failures_then_success_reset_and_retain_evidence (tools.test_track_a_recovery.RecoveryTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 28 tests in 0.319s
|
||||
|
||||
OK
|
||||
@@ -691,3 +691,13 @@ Seed1 postfit实际独立审阅采用:Assist1794993449的7b4da089/51c2514c原
|
||||
## Track A唯一基础设施恢复准备获准(2026-11-18)
|
||||
|
||||
Claude1795000465/0626允许一次标记恢复:0–2hash复用,3/4同seed0配方fresh,旧47943.947195一次,总162000/新train106856.052805/post7200。仅新入口10秒GPU查询、3连续采样失败容错且全部留日志、实际资源违规立即失败、单job失败隔离;seed1不动、真实idle、无进一步恢复。独立c5a67bd4/d367c4e0已固定、21CPU合成PASS,1795001544送Assist窄delta;无review采用/执行。原postfit983bba99的固定旧root/合同与mixed模型不兼容,已向Claude明确:只可保留原统计做独立身份绑定,缺折不评分,不伪造旧PASS跨绑定。原STOP、测试失败、旧CPU记录缺口不改。
|
||||
|
||||
|
||||
## 最新 Track A recovery 两项边界修复固定,待同次定点复核(2026-11-18)
|
||||
|
||||
原Assist1795001813 BLOCKED(2bd48504)已实读并保留,原两个AST反例已复现。仅common/queue/guard/test四文件修复;28CPU检查PASS,科学worker、原Track A与seed1 pins保持。新implementation `8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b`,contract `4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa`,proof `8baec54ffea1b51c96de857072c4a6cb074792626d9e679e29fd3390b2ffc048`,comparison828530e4不变,27code/60source。完整binding SHA `8c13e9ebde5793cc5f25965e5f280c87cd0310f22f19eeaae7a8992c70bcc3c7`。config/reuse_identity与新五字段实核PASS;原47文件快照和失败证据完整保留。
|
||||
|
||||
局部停止折先own_stop,再把已拒绝未来训练排除总投影;保留原raw投影及max(actual,原单折30960)清理储备,终态替换actual,真实累计超帽仍全局停止。prestart查询前含10秒查询+20秒清理,查询后/睡眠前/worker前核截止;两失败一idle重复也按时停止且不启动worker。旧47943.947195一次/总162000/post7200不变,不加帽不retry。详见docs/track-a-recovery-boundary-fix-2026-11-18.md。
|
||||
|
||||
已向原Slack串提交并请TRON原生转Assist同次两项delta复核;尚无新实际PASS/review采用/started/训练,不声称Assist已实收。Claude1566全五完成后identity-only postfit新绑定裁量保持,原统计不改,缺折不得评分。原STOP不重启,seed1原队列不动。下方旧c5a67bd4待审记录为历史,不能启动旧版。
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
## 最新 Track A recovery 两项边界修复固定,待同次定点复核(2026-11-18)
|
||||
|
||||
原Assist1795001813 BLOCKED(2bd48504)已实读并保留,原两个AST反例已复现。仅common/queue/guard/test四文件修复;28CPU检查PASS,科学worker、原Track A与seed1 pins保持。新implementation `8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b`,contract `4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa`,proof `8baec54ffea1b51c96de857072c4a6cb074792626d9e679e29fd3390b2ffc048`,comparison828530e4不变,27code/60source。完整binding SHA `8c13e9ebde5793cc5f25965e5f280c87cd0310f22f19eeaae7a8992c70bcc3c7`。config/reuse_identity与新五字段实核PASS;原47文件快照和失败证据完整保留。
|
||||
|
||||
局部停止折先own_stop,再把已拒绝未来训练排除总投影;保留原raw投影及max(actual,原单折30960)清理储备,终态替换actual,真实累计超帽仍全局停止。prestart查询前含10秒查询+20秒清理,查询后/睡眠前/worker前核截止;两失败一idle重复也按时停止且不启动worker。旧47943.947195一次/总162000/post7200不变,不加帽不retry。详见docs/track-a-recovery-boundary-fix-2026-11-18.md。
|
||||
|
||||
已向原Slack串提交并请TRON原生转Assist同次两项delta复核;尚无新实际PASS/review采用/started/训练,不声称Assist已实收。Claude1566全五完成后identity-only postfit新绑定裁量保持,原统计不改,缺折不得评分。原STOP不重启,seed1原队列不动。下方旧c5a67bd4待审记录为历史,不能启动旧版。
|
||||
|
||||
## 最新 Track A唯一标记recovery绑定已提交窄审,未运行(2026-11-18)
|
||||
|
||||
Claude1795000465/0626已实读,允许原基础设施abort的一次新root恢复。root `/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18`;implementationc5a67bd4/contractd367c4e0/prooffef611c1/comparison828530e4/27codes/60sources,完整binding99a5836d。21CPU合成PASS,原1个测试fixture RSS失败与诊断保留,worker科学AST保持。1795001544已提交并请TRON原生转Assist窄delta审阅;尚无实际新review/started/waiter/训练。当前目标仅binding posted,不复制或自动启动。
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# Track A recovery:同次窄审两项边界修复
|
||||
|
||||
MeltAssist1795001813.749119实际BLOCKED两项控制流边界。已直接读取review/evidence/reproducer并核SHA:review `2bd485043af0ebb42aab3bd93da0eedddcb706f652a2868280f011ee486c6599`;evidence `a6c9eb98ca7d3f5e95e0b548aafcea3d980477ffba11ebb19c4809bc7d1ba386`;reproducer `a01d403f400b4adabfca32c3291dfd6228878634b963353bccd274a4924da696`。原c5a67bd4/d367c4e0绑定未获运行许可,始终没有started/worker/模型/预测。
|
||||
|
||||
修前47文件逐字快照(原代码、合同、proof、binding、BLOCKED、复现脚本/证据)保存在`artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation`,preservation.json逐文件校验。原AST反例已在修改前实跑,输出original-reproduction.json保留;这是合成控制流复现,不是实际训练失败。
|
||||
|
||||
## RECOVERY-OWN-PROJECTION-ISOLATION
|
||||
|
||||
原反例job3首epoch1000s、actual160、单折未来投影249030,先放入总投影后再own_stop,导致总335133.947195错误触发global-stop。
|
||||
|
||||
修复顺序为:保存原cell投影→先写本折stop→从已有stop文件明确局部停止集合→计算仍会执行的未来工作。对局部停止、尚未终态的折,不再计入已拒绝的完整未来训练;保守保留`max(actual, 原批准单折30960)`作为终止清理窗口,直至真实终态以实际成本替换。原raw cell/estimates不改,不抹失败投影或actual;没有增加任何帽、没有重试。持续运行折仍按原最大epoch×1.25/actual floor规则。真实累计actual、共享源或资源损坏仍按原队列全局保护。
|
||||
|
||||
新实际epoch块AST检查:job3原249030投影与失败判定保留,仅own-stop;暂按它原30960整帽+健康另一折30960+post7200+old47943.947195,累计117063.947195仍可继续。健康job4有实际新epoch时自身门PASS。另验停止折actual超过自己帽仍完整记账,终态actual只替换一次;真实累计超162000仍global-stop。
|
||||
|
||||
## RECOVERY-PRESTART-DEADLINE
|
||||
|
||||
原already-expired elapsed30961/remaining30960仍发14次合成query且0次elapsed检查的反例已保留。
|
||||
|
||||
新增require_prestart_window,针对已扣派发前等待的本折剩余cap:查询前预留10秒query+20秒清理;查询后再次检查;每次0.5秒错误等待或2秒idle间隔前预留该sleep+20秒清理;三idle后启动watcher/worker前也再检查。等号即拒绝,非有限/负数时钟拒绝。过期只生成本fold未启动失败,沿既有collect隔离,不触发兄弟折停止。每次失败样本仍由原Monitor记原文,三连续规则、有效成功复位与缺样本null保持。
|
||||
|
||||
实际idle-loop AST配合fake clock:已经过期时0query;反复两次失败/一次idle成功,在未出现三连续错误时也按窗口停止,worker不会启动;额外覆盖查询后超窗及sleep之前不足。没有GPU查询或科学数据。
|
||||
|
||||
## 范围与验证
|
||||
|
||||
只变更common、queue、guard和test四个恢复文件。worker、monitor、supervisor、原科学代码、seed1及原Track A pins保持。comparison `828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b`和原资源帽/旧账/复用名单不变;新完整五字段在同root的expected-review-binding.json,须本次实际新PASS后才能启动。
|
||||
|
||||
28项CPU合成PASS(原21+7项边界回归),测试日志位于boundary-fix目录;原测试fixture失败与原BLOCKED仍保存。没有真实训练/恢复/模型加载/测试目标/预测评分。原NO_RETRY及无进一步恢复规则不变。
|
||||
|
||||
另已直接读Claude1795001566.514629:接受原postfit983bba99与新mixed来源不兼容,完整五折之后需identity-only新绑定及MeltAssist窄delta,原统计不改、缺折不得绕过完整性门。本次不实现或启动postfit。
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"status": "FIXED_PENDING_SAME_REVIEW",
|
||||
"unix": 1795002168.5627744,
|
||||
"changed_code_files": [
|
||||
"tools/test_track_a_recovery.py",
|
||||
"tools/track_a_recovery_common.py",
|
||||
"tools/track_a_recovery_guard.py",
|
||||
"tools/track_a_recovery_queue.py"
|
||||
],
|
||||
"unchanged_code_files": 23,
|
||||
"source_pins": "PASS",
|
||||
"original_track_a_pins": "PASS",
|
||||
"seed1_pins": "PASS",
|
||||
"comparison_unchanged": true,
|
||||
"scientific_worker_unchanged": true,
|
||||
"cpu_tests": 28,
|
||||
"tests_pass": true,
|
||||
"tests_log_sha256": "84627ca57ea0e69e3b67e3aa8e2907c670e45962bd0fd3686c4f24ca0beb5fb4",
|
||||
"old_reproduction_sha256": "fe4be891c78e2fe179234c891609b0b9c7c021e30fec43d75de23a79674c851c",
|
||||
"preservation_sha256": "1c4d1b5c1de29d0e6707767115535cd3f23c5063534df323baf00747dacda076",
|
||||
"no_execution": true
|
||||
}
|
||||
@@ -21336,13 +21336,13 @@
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "108e4c27d9743d30baa8faf9c16ab6b139d911a5a72f00db7a4df7334f8d122a",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "63364eda4f0fbb0d288b71f359661e6c004aab5443b17dd7f15570cce4f4efb0",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "d35e57de98ec0f4a3d40232925fe5d31bf05ebc93855e6b1e7d9fe0592cdfba9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5"
|
||||
},
|
||||
@@ -21418,5 +21418,11 @@
|
||||
"gpu_query_timeout_seconds": 10,
|
||||
"consecutive_failed_samples": 3,
|
||||
"job_failure_isolated": true,
|
||||
"no_auto_launch": true
|
||||
"no_auto_launch": true,
|
||||
"boundary_fix": {
|
||||
"review_sha256": "2bd485043af0ebb42aab3bd93da0eedddcb706f652a2868280f011ee486c6599",
|
||||
"old_implementation": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"own_stop_reserve": "max(actual,original_own_cap) until actual terminal; raw rejected projection preserved",
|
||||
"prestart": "before/after queries, before sleeps and before worker;10s query+20s cleanup in remaining cap"
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"reviewed_commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"implementation_proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"reviewed_commit": "8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b",
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"implementation_proof_sha256": "8baec54ffea1b51c96de857072c4a6cb074792626d9e679e29fd3390b2ffc048",
|
||||
"comparison_contract_sha256": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
@@ -22,13 +22,13 @@
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "108e4c27d9743d30baa8faf9c16ab6b139d911a5a72f00db7a4df7334f8d122a",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "63364eda4f0fbb0d288b71f359661e6c004aab5443b17dd7f15570cce4f4efb0",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "d35e57de98ec0f4a3d40232925fe5d31bf05ebc93855e6b1e7d9fe0592cdfba9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5"
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"proof_sha256": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2"
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"proof_sha256": "8baec54ffea1b51c96de857072c4a6cb074792626d9e679e29fd3390b2ffc048"
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
{
|
||||
"commit": "c5a67bd4e420cd1ddb85101dacbf3a10f073e0e9",
|
||||
"commit": "8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/comparison-contract.json": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "7002cfc60511d80f6aae331d255fad501eeaaa32b63f1d7aa29b36e5b2558fae",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/boundary-fix-audit.json": "69093ca90bc827703fed5db1e7a4cdc7145cb5813b06a3a87e13013ed36ca9fe",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-entry-2026-11-18.md": "070536c0addeeef2d2ce70edb6387f26307fad5fc1cd0df22fec0def36398732",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-boundary-fix-2026-11-18.md": "a9b3c3c6418057b8cbb5e6678496c75db925d03450f1f610e11b8f285650b2d2",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
@@ -23,30 +25,73 @@
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_recovery.py": "108e4c27d9743d30baa8faf9c16ab6b139d911a5a72f00db7a4df7334f8d122a",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_guard.py": "63364eda4f0fbb0d288b71f359661e6c004aab5443b17dd7f15570cce4f4efb0",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "d35e57de98ec0f4a3d40232925fe5d31bf05ebc93855e6b1e7d9fe0592cdfba9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "b5e937c870225059ec38fbefbb2b933ef46cea10fb38fe520156749da37cf3c8",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "7db6ca25fbc9586e893065728b955c72877f5f16dc43e455f1b83eeb9a576fdf",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "85483476d5bfa7274b5a3d658fe5e57227d26063b56a6827f6a9fb5960715b4d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "b967bd9e67c37f99af42b8160941dc5899a5193cc15bda243e2eaa79c5100434",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "4027923ffc3bd93b1b9cea875cbba236b56249e55a41af354cf19f226af6c441",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "c8aa937dec2e2ca9dfdaa102c0a61e4cb11b513d7ded085887b471e6d0a2478d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "805226f37c52f96a095566dd6ee5f1e4c20871ebada2db99af679c6d197458da"
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/preservation.json": "1c4d1b5c1de29d0e6707767115535cd3f23c5063534df323baf00747dacda076",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-reproduction.json": "fe4be891c78e2fe179234c891609b0b9c7c021e30fec43d75de23a79674c851c",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/tests-attempt-01.log": "84627ca57ea0e69e3b67e3aa8e2907c670e45962bd0fd3686c4f24ca0beb5fb4",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-git-freeze.json": "fef611c113f5141397baa6c83ca0123f4024be3ada2f6659d7827e4d019c5ab2",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "7002cfc60511d80f6aae331d255fad501eeaaa32b63f1d7aa29b36e5b2558fae",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/expected-review-binding.json": "99a5836d5d14082dfbacb560637dda4270566837031fa9b902b9805a61f8eff0",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/binding-audit.json": "5a327e78162bfd94e985c0f132e1aa169b99c1b1f74088ac8ff11a00835df449",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-authorization.json": "a3fa7bfc860754f469f967372f0e2100cb9bde29e7ce2b45efe19ec07b89ea16",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/contract.json": "d367c4e089d0bf89277fdde542a7b64745ecaada379f36c7b33a6ad3d8b9a174",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/comparison-contract.json": "828530e4a02e70f81172fede011788f4ad90c6b01060d2a014ce38898b16364b",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/prepare_track_a_recovery.py": "f1d7cc6d1f073c8f41063f82a620abcca810d0595a8d2dbacdf1c5e03aa634c6",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/test_track_a_recovery.py": "747f09aa5e0c51f33f30b52c371b0340545cfd4760f009d6f91691f603dd19a1",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_worker.py": "82944fc3aa113a6005999449faf1c10635ff19073c7eb8fa682967725bc206e5",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_queue.py": "59aa0cb70918086f387786a33338448a50900e14133da9c2a05af391c8c602e4",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_common.py": "a1e1c05b44eb69f205ab22efeaba5c7414cc2c112eb0083a57849a6635098ccc",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_guard.py": "84b9f8786900654a5d97711beb828c09b392b9410f1bccccbd70820859d72e91",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/docs/track-a-recovery-entry-2026-11-18.md": "070536c0addeeef2d2ce70edb6387f26307fad5fc1cd0df22fec0def36398732",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "b5e937c870225059ec38fbefbb2b933ef46cea10fb38fe520156749da37cf3c8",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "7db6ca25fbc9586e893065728b955c72877f5f16dc43e455f1b83eeb9a576fdf",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "85483476d5bfa7274b5a3d658fe5e57227d26063b56a6827f6a9fb5960715b4d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "b967bd9e67c37f99af42b8160941dc5899a5193cc15bda243e2eaa79c5100434",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "4027923ffc3bd93b1b9cea875cbba236b56249e55a41af354cf19f226af6c441",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "c8aa937dec2e2ca9dfdaa102c0a61e4cb11b513d7ded085887b471e6d0a2478d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "805226f37c52f96a095566dd6ee5f1e4c20871ebada2db99af679c6d197458da",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json": "a6c9eb98ca7d3f5e95e0b548aafcea3d980477ffba11ebb19c4809bc7d1ba386",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py": "a01d403f400b4adabfca32c3291dfd6228878634b963353bccd274a4924da696",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/review.json": "2bd485043af0ebb42aab3bd93da0eedddcb706f652a2868280f011ee486c6599"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/comparison-contract.json": "home-runs/track-a-recovery-2026-11-18/comparison-contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "home-runs/track-a-recovery-2026-11-18/preparation-receipt.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/boundary-fix-audit.json": "home-runs/track-a-recovery-2026-11-18/boundary-fix-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-entry-2026-11-18.md": "docs/track-a-recovery-entry-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-recovery-boundary-fix-2026-11-18.md": "docs/track-a-recovery-boundary-fix-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
@@ -74,15 +119,56 @@
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py": "tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_worker.py": "tools/track_a_recovery_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl"
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/preservation.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/preservation.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-reproduction.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-reproduction.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/tests-attempt-01.log": "artifacts/track-a-recovery-boundary-fix-2026-11-18/tests-attempt-01.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-git-freeze.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-git-freeze.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/preparation-receipt.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/expected-review-binding.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/expected-review-binding.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/binding-audit.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/binding-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-authorization.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/implementation-authorization.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/contract.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/comparison-contract.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/home-runs/track-a-recovery-2026-11-18/comparison-contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/prepare_track_a_recovery.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/prepare_track_a_recovery.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/test_track_a_recovery.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/test_track_a_recovery.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_worker.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_queue.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_common.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_common.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_guard.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/gre24_models.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_monitor.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_supervisor.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/docs/track-a-recovery-entry-2026-11-18.md": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/docs/track-a-recovery-entry-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/resource-snapshot.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-02.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/attempt-01.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/execution.lock",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/ledger.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.started.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/track-a-recovery-acceptance-2026-11-18/transient-case-diagnostic/synthetic.monitor-errors.jsonl",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/evidence.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/reproduce_boundaries.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/review.json": "artifacts/track-a-recovery-boundary-fix-2026-11-18/original-implementation/artifacts/meltassist-track-a-recovery-review-2026-11-18/review.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
|
||||
@@ -133,6 +133,81 @@ class RecoveryTests(unittest.TestCase):
|
||||
ns['dispatch']();ns['dispatch']();self.assertFalse(calls);ns['dispatch']()
|
||||
self.assertEqual(calls,[(4,'THREE_FAILED_DISPATCH_GPU_SAMPLES')]);self.assertIs(active[3],other)
|
||||
|
||||
def test_extreme_rejected_projection_cannot_stop_affordable_survivor(self):
|
||||
from types import SimpleNamespace
|
||||
from track_a_recovery_common import project_cell
|
||||
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);save(root/'contract.json',{})
|
||||
for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True)
|
||||
save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000))
|
||||
save(root/'jobs/4/worker/history/001.json',dict(epoch=1,train_epoch_seconds=50))
|
||||
own=[];glob=[]
|
||||
def stop(i,why):own.append((i,why));save(root/'jobs'/str(i)/'stop.json',dict(reason=why))
|
||||
ns=dict(root=root,c=self.cfg(),active={3:object(),4:object()},terminal={},estimates={},last_epoch={},read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,time=SimpleNamespace(time=lambda:0),own_stop=stop,global_stop=glob.append)
|
||||
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_epoch_block','exec'),ns)
|
||||
self.assertEqual(own,[(3,'OWN_PROJECTED_CAP')]);self.assertFalse(glob)
|
||||
failed=json.loads((root/'jobs/3/first-epoch-projection.json').read_text());healthy=json.loads((root/'jobs/4/first-epoch-projection.json').read_text())
|
||||
self.assertEqual(failed['cell']['projected_gpu_seconds'],249030);self.assertEqual(failed['status'],'STOP_OWN_COST_PROJECTION')
|
||||
self.assertEqual(failed['total']['cumulative_projected_gpu_seconds'],OLD+30960+30960+7200)
|
||||
self.assertEqual(healthy['status'],'PASS');self.assertFalse((root/'jobs/4/stop.json').exists())
|
||||
self.assertEqual(ns['estimates'][3]['projected_gpu_seconds'],249030)
|
||||
|
||||
def test_stopping_reserve_keeps_actual_and_terminal_settles_once(self):
|
||||
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:40000},stopping={3})
|
||||
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],40000)
|
||||
self.assertEqual(p['rows'][0]['basis'],'locally_stopping_own_cap_or_actual')
|
||||
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)},{3:dict(gpu_seconds_charged=190)}, {},stopping={3})
|
||||
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],190);self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
|
||||
|
||||
def test_true_cumulative_overrun_still_stops_after_local_rejection(self):
|
||||
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:130000,4:500},stopping={3})
|
||||
self.assertEqual(p['status'],'STOP_CUMULATIVE_BUDGET');self.assertGreater(p['cumulative_projected_gpu_seconds'],162000)
|
||||
|
||||
def idle_replay(self,clock,query,cap,sleep):
|
||||
from types import SimpleNamespace
|
||||
from track_a_recovery_guard import require_prestart_window
|
||||
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);monitor=Monitor(root/'errors.jsonl')
|
||||
ns=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=cap),elapsed=clock,require_prestart_window=require_prestart_window,time=SimpleNamespace(time=clock,sleep=sleep),save=save)
|
||||
started=False
|
||||
try:
|
||||
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_idle_loop','exec'),ns)
|
||||
started=True
|
||||
except RuntimeError as exc:reason=str(exc)
|
||||
else:reason=None
|
||||
return dict(worker_would_start=started,reason=reason,policy=monitor.policy(),errors=list(monitor.records),successes=ns['success'])
|
||||
|
||||
def test_already_expired_prestart_makes_zero_queries(self):
|
||||
calls=[]
|
||||
d=self.idle_replay(lambda:30961,lambda u:calls.append(u),30960,lambda s:None)
|
||||
self.assertEqual(calls,[]);self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
||||
|
||||
def test_alternating_two_errors_one_idle_expires_without_worker(self):
|
||||
clock=[60.];calls=[]
|
||||
def query(u):
|
||||
calls.append(u);clock[0]+=10
|
||||
if len(calls)%3:raise TimeoutError('synthetic recoverable')
|
||||
return 0,0
|
||||
def sleep(s):clock[0]+=s
|
||||
d=self.idle_replay(lambda:clock[0],query,160,sleep)
|
||||
self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
||||
self.assertFalse(d['policy']['failed']);self.assertEqual(d['policy']['max_streaks']['gpu_usage'],2)
|
||||
self.assertLess(clock[0],160);self.assertLessEqual(len(calls),7);self.assertGreater(len(d['errors']),3)
|
||||
|
||||
def test_prestart_checks_after_query_overrun(self):
|
||||
clock=[60.];calls=[]
|
||||
def query(u):calls.append(u);clock[0]=85;return 0,0
|
||||
d=self.idle_replay(lambda:clock[0],query,100,lambda s:None)
|
||||
self.assertFalse(d['worker_would_start']);self.assertEqual(len(calls),1);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
||||
|
||||
def test_prestart_checks_before_idle_sleep(self):
|
||||
clock=[60.];sleeps=[]
|
||||
def query(u):clock[0]=79.5;return 0,0
|
||||
d=self.idle_replay(lambda:clock[0],query,100,sleeps.append)
|
||||
self.assertFalse(d['worker_would_start']);self.assertEqual(sleeps,[]);self.assertEqual(d['successes'],1)
|
||||
|
||||
def test_fast_local_guard_not_blocked_on_gpu_query(self):
|
||||
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());local=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='local_watch')
|
||||
self.assertNotIn('device_usage',ast.unparse(local));self.assertIn('WHOLE_TREE_RSS_LIMIT',ast.unparse(local));self.assertIn('OUTPUT_DISK_LIMIT',ast.unparse(local));self.assertIn('RECORDED_VRAM_LIMIT',(B/'tools/track_a_recovery_guard.py').read_text())
|
||||
|
||||
@@ -76,13 +76,21 @@ def accepted_guard(rc,g):
|
||||
and not w['monitor_policy']['failed'] and w['monitor_policy']['errors']==w['monitor_errors'])
|
||||
|
||||
|
||||
def project_total(c,estimates,terminal,live):
|
||||
if (set(estimates)|set(terminal)|set(live))-set(RECOVERY):raise ValueError('Unknown recovery fold')
|
||||
def project_total(c,estimates,terminal,live,stopping=None):
|
||||
stopping=set(stopping or [])
|
||||
if (set(estimates)|set(terminal)|set(live)|stopping)-set(RECOVERY):raise ValueError('Unknown recovery fold')
|
||||
rows=[];new=c['postfit_gpu_reserve']
|
||||
for i in RECOVERY:
|
||||
value=terminal[i]['gpu_seconds_charged'] if i in terminal else max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']))
|
||||
if i in terminal:
|
||||
value=terminal[i]['gpu_seconds_charged'];basis='terminal_actual'
|
||||
elif i in stopping:
|
||||
# No rejected future training remains scheduled. Retain the full
|
||||
# originally approved own cap for bounded cleanup until actual exit.
|
||||
value=max(live.get(i,0),c['jobs'][i]['gpu_seconds']);basis='locally_stopping_own_cap_or_actual'
|
||||
else:
|
||||
value=max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']));basis='continuing_projection_or_pending_cap'
|
||||
if not math.isfinite(value) or value<0:raise ValueError('Invalid new charge')
|
||||
new+=value;rows.append(dict(id=i,projected_gpu_seconds=value))
|
||||
new+=value;rows.append(dict(id=i,projected_gpu_seconds=value,basis=basis))
|
||||
# Per-cell excess is isolated by own stop.json; total exhaustion stays global.
|
||||
return dict(status='PASS' if OLD+new<=c['gpu_cap'] else 'STOP_CUMULATIVE_BUDGET',rows=rows,new_projected_gpu_seconds=new,old_gpu_seconds_once=OLD,cumulative_projected_gpu_seconds=OLD+new,postfit_reserve=c['postfit_gpu_reserve'])
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Isolated recovery fold: 10s GPU samples, three failures; fast local guards stay live."""
|
||||
import fcntl,json,os,resource,signal,sys,threading,time
|
||||
import fcntl,json,math,os,resource,signal,sys,threading,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
@@ -9,6 +9,15 @@ from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usa
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
|
||||
def require_prestart_window(elapsed,cap,next_seconds=0):
|
||||
"""Reserve bounded query/sleep plus20s cleanup inside the remaining own cap."""
|
||||
if any(not math.isfinite(v) or v<0 for v in [elapsed,cap,next_seconds]):
|
||||
raise ValueError('Invalid prestart deadline')
|
||||
if elapsed+next_seconds+20>=cap:
|
||||
raise RuntimeError('PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
||||
|
||||
|
||||
def run(queue,fold):
|
||||
contract,_=config(queue)
|
||||
if fold not in contract['recovery_folds']:raise PermissionError('Only fresh folds3/4')
|
||||
@@ -62,16 +71,22 @@ def run(queue,fold):
|
||||
samples=[];success=0
|
||||
while success<3:
|
||||
if requested or (root/'stop.json').exists() or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_START')
|
||||
require_prestart_window(elapsed(),job['gpu_seconds'],10)
|
||||
value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid']))
|
||||
require_prestart_window(elapsed(),job['gpu_seconds'])
|
||||
if not ok:
|
||||
success=0
|
||||
if monitor.failed:raise RuntimeError('THREE_CONSECUTIVE_IDLE_SAMPLE_FAILURES')
|
||||
require_prestart_window(elapsed(),job['gpu_seconds'],.5)
|
||||
time.sleep(.5);continue
|
||||
samples.append(dict(unix=time.time(),memory_bytes=value[0],utilization=value[1]))
|
||||
save(root/'idle-polls.json',samples)
|
||||
if value[0]>128*2**20 or value[1]!=0:raise RuntimeError('SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED')
|
||||
success+=1
|
||||
if success<3:time.sleep(2)
|
||||
if success<3:
|
||||
require_prestart_window(elapsed(),job['gpu_seconds'],2)
|
||||
time.sleep(2)
|
||||
require_prestart_window(elapsed(),job['gpu_seconds'])
|
||||
for fn in [local_watch,gpu_watch]:
|
||||
t=threading.Thread(target=fn,daemon=True);t.start();threads.append(t)
|
||||
remaining=job['gpu_seconds']-elapsed()-20
|
||||
|
||||
@@ -89,11 +89,15 @@ def run(root):
|
||||
rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))]
|
||||
if rows and rows[-1]['epoch']>last_epoch.get(i,0):
|
||||
epoch=rows[-1]['epoch'];cell=project_cell(c['jobs'][i],dict(epoch=epoch,max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),charged=60+age()))
|
||||
estimates[i]=cell;total=project_total(c,estimates,terminal,live())
|
||||
estimates[i]=cell
|
||||
# Isolate the rejected cell before budgeting future work.
|
||||
# Its raw estimate is retained in estimates/result unchanged.
|
||||
if not cell['within_cap']:own_stop(i,'OWN_PROJECTED_CAP')
|
||||
stopping={j for j in active if (root/'jobs'/str(j)/'stop.json').exists()}
|
||||
total=project_total(c,estimates,terminal,live(),stopping=stopping)
|
||||
result=dict(status='PASS' if cell['within_cap'] and total['status']=='PASS' else 'STOP_OWN_COST_PROJECTION',cell=cell,total=total,contract_sha256=sha(root/'contract.json'),unix=time.time())
|
||||
name='first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
||||
save(root/'jobs'/str(i)/name,result);save(root/'latest-cost-projection.json',total);last_epoch[i]=epoch
|
||||
if not cell['within_cap']:own_stop(i,'OWN_PROJECTED_CAP')
|
||||
if total['status']!='PASS':global_stop('CUMULATIVE_PROJECTED_CAP')
|
||||
if not (root/'global-stop.json').exists():dispatch()
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user