Prepare one labelled item1 postfit recovery with bounded idle wait

This commit is contained in:
MeltAstra committed 2026-11-18 14:04:43 +08:00
1 parent 4f646232f4
commit 581208098f
24 files changed
+83789

No files matched your search

@@ -0,0 +1,12 @@
{
"status": "PASS",
"tests": 15,
"failures": 0,
"errors": 0,
"cpu_seconds": 2.229610314,
"wall_seconds": 2.240639487048611,
"source_sha256": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
"GPU_used": false,
"real_scientific_arrays_read": false,
"attempt01_preserved": true
}
@@ -0,0 +1,125 @@
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
import copy
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save, sha
from phase3_item1_postfit_common import read, require_budget
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
from phase3_item1_resume import install_reuse
import phase3_item1_resume_guard as guard
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.old = read(O / 'contract.json')
self.done = read(O / 'complete.json')
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
training_gpu_once=self.done['training_gpu_seconds_once'])
def test_scope_and_actual_original_stop(self):
validate_scope(self.c, self.old)
validate_stop(self.done)
def test_no_predict0_rerun(self):
self.c['resume_indices'] = list(range(7))
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_gpu1_only(self):
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
self.c['gpu_uuid'] = uuid
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_no_budget_change(self):
self.c['postfit_gpu_cap'] = 7201
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
def test_failed_job_worker_must_be_null(self):
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
with self.assertRaises(ValueError): validate_stop(self.done)
def test_predict0_must_be_pass(self):
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
with self.assertRaises(ValueError): validate_stop(self.done)
def test_old_cost_once_not_per_job(self):
a = cumulative_cost(self.c, 100., 200.)
b = cumulative_cost(self.c, 300., 500.)
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
def test_old_cost_limits_remaining_reserve(self):
d = cumulative_cost(self.c,60,180)
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
with self.assertRaises(RuntimeError):
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
def test_original_six_receipts_currently_clean(self):
self.assertEqual(len(six_receipts(self.c)['rows']),6)
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
with self.assertRaises(FileExistsError):install_reuse(root,[e])
def test_reuse_bad_hash_rejected_before_write(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
with self.assertRaises(ValueError):
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
self.assertEqual(list(root.iterdir()),[])
def guarded(self, mode):
timer=[0.];calls=[0]
def sleep(seconds): timer[0]+=seconds
def probe(uuid):
calls[0]+=1
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
return 0,0
class Thread:
def __init__(self,*a,**k):pass
def start(self):pass
def join(self,**k):pass
def is_alive(self):return False
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
def supervisor(*a,**k):
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
result=guard.run_prediction(root,c,{},1,0,300)
return result,sup.call_count,read(out/'recovery-idle-wait.json')
def test_busy_then_idle_counts_wait_and_three_polls(self):
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
self.assertEqual(r['gpu_seconds_charged'],65.)
self.assertEqual(len(w['samples']),3)
def test_busy_timeout_stops_without_worker(self):
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
self.assertEqual(r['gpu_seconds_charged'],80.)
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
def test_supervisor_monitor_error_still_rejected(self):
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
def test_wait_limit_cannot_be_relaxed(self):
self.c['idle_wait_seconds']=21
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
if __name__ == '__main__':unittest.main()
@@ -0,0 +1,10 @@
{
"status": "FAIL",
"tests": 15,
"failures": 1,
"errors": 0,
"unittest_wall_seconds": 1.478,
"test_source_sha256": "0835350eae48f3766d8bf34caa58eefe2d0670e5ebea092a47f7df744877c4ea",
"failure": "test_old_cost_limits_remaining_reserve expected7 remaining to exceed cap at newCPU180, but total3570.833960341 remains below3600. Incorrect synthetic assertion; no production budget change.",
"real_scientific_execution": false
}
@@ -0,0 +1,20 @@
test_busy_then_idle_counts_wait_and_three_polls (test_phase3_item1_resume.RecoveryTests) ... ok
test_busy_timeout_stops_without_worker (test_phase3_item1_resume.RecoveryTests) ... ok
test_failed_job_worker_must_be_null (test_phase3_item1_resume.RecoveryTests) ... ok
test_gpu1_only (test_phase3_item1_resume.RecoveryTests) ... ok
test_no_budget_change (test_phase3_item1_resume.RecoveryTests) ... ok
test_no_predict0_rerun (test_phase3_item1_resume.RecoveryTests) ... ok
test_old_cost_limits_remaining_reserve (test_phase3_item1_resume.RecoveryTests) ... ok
test_old_cost_once_not_per_job (test_phase3_item1_resume.RecoveryTests) ... ok
test_original_six_receipts_currently_clean (test_phase3_item1_resume.RecoveryTests) ... ok
test_predict0_must_be_pass (test_phase3_item1_resume.RecoveryTests) ... ok
test_reuse_bad_hash_rejected_before_write (test_phase3_item1_resume.RecoveryTests) ... ok
test_reuse_copy_preserves_original_and_rejects_duplicate (test_phase3_item1_resume.RecoveryTests) ... ok
test_scope_and_actual_original_stop (test_phase3_item1_resume.RecoveryTests) ... ok
test_supervisor_monitor_error_still_rejected (test_phase3_item1_resume.RecoveryTests) ... ok
test_wait_limit_cannot_be_relaxed (test_phase3_item1_resume.RecoveryTests) ... ok
----------------------------------------------------------------------
Ran 15 tests in 2.240s
OK
+8
View File
@@ -1,3 +1,11 @@
## 最新 item1 后处理最小恢复已固定待窄审(2026-11-18)
已实读Claude0643/9347及1566:原postfit STOP保留,predict0的686文件逐字hash复用,新根phase3-item1-postfit-recovery-2026-11-18只predict1–6;GPU1only、每job20s等待后原三idle polls,全部在900s及总账内,无换卡/重试。实现8a3962b4/contract7e991ea1/proofa8e671ff,comparison332f2515不变,54pins含全部原47不变。15CPU合成PASS,首测试断言失败/源已保留;尚无review/started/新预测。旧GPU261.150194630/CPU270.833960341/训练71192.071785253各计一次。详见docs/phase3-item1-postfit-recovery-2026-11-18.md;须Assist窄delta后执行,旧PASS不作新审阅。Track A原队列不改、2/3保留。
## 最新 Track A 首两个epoch成本门PASS(2026-11-18)
原唯一1262364继续,job0/1 worker1262432/1262422已进入epoch2;完整首epoch61.848708608s/59.255394293s,逐job门均PASS。最新总投影130716.145896GPU-s/162000,含未启动2/3/4整帽和7200s postfit预留,不是全5测量。source6701e4d6/contractbe4ac1f6不变,无重启/改源码。审阅收讫+实际启动+两门证据已远端固定4f646232,报告和原始first-two-epoch-audit.json保留。队列只checkpoint后停,test/predict/score需后续原pre-score审阅。
## 最新 Track A H2 唯一训练已实际启动(2026-11-18 12:28)
Assist1794975952原件1fa923f8已逐字接收/固定b7879eec并validate PASS。唯一service meltmind-track-a-h2-20261118.service/coord1262364,root /home/frank14f/MeltMind-runs/track-a-h2-2026-11-18;job0 guard1262373/worker1262432 GPU2,job1 guard1262374/worker1262422 GPU3,首epoch真实51/311和58/304快照,三idle polls各PASS。source6701e4d6/contractbe4ac1f6不变,23pins PASS。job2/3/4 pending,162000GPU-s含7200postfit预留,各job首epoch成本门自动执行,尚未成本PASS;禁止重复启动/改运行源。item1旧worker1135393仍GPU1(56503/64000),GPU0保留其postfit。队列只freeze模型后结束,test/predict/score仍须原独立pre-score审阅,训练PASS不得替代。实际证据actual-start-audit.json。下方未启动/待审仅历史。
@@ -0,0 +1,23 @@
# Item1 postfit: one labelled infrastructure continuation
Prepared and locally frozen; narrow delta review pending, no new prediction or scoring started. Claude1794979347/1794980643 authorize reuse of predict0 and exactly predict1–6; Claude1794981566 fixes GPU1 only and the bounded20s idle wait. The original STOP is infrastructure-labelled with cause undiagnosed, not a scientific failure or a proven utilization-sampling issue.
Original root `/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18` remains untouched. Its complete SHA8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8 records predict0 PASS, predict1 stopped before any worker (`worker=null`), GPU261.1501946300268s and CPU270.833960341s. Training71192.071785253GPU-s remains a separate once-only amount. Original started, resource-progress, six-receipt audit and raw guards are retained.
New root `/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18`. Implementation8a3962b4c1267a7f021e392098098fb7ff20f562; contract7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622; proofa8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f. Original comparison332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9 stays byte-identical. expected-review-binding.json includes54 code pins; all original47 pins remain identical. This new delta does not reuse old PASS75bd9ff1 as its own verdict.
New files are `phase3_item1_resume.py`, `phase3_item1_resume_common.py`, `phase3_item1_resume_guard.py`, `prepare_phase3_item1_resume.py`, and `test_phase3_item1_resume.py`. The original prediction worker, feature loader, score, statistics and comparison functions are imported/run unchanged. The independent coordinator uses the new root and original seven endpoint metadata, but dispatches only indices1..6. No training, model-selection or field-decoding step is added.
Reuse: all686 predict0 files (206411454bytes) have their original manifest hashes checked as opaque bytes; no arrays or model values were decoded during preparation. The old manifest has complete epoch25 and c25F coverage over343parts. Model adoption is independently matched to the old adopted-model manifest and predict0 environment checkpoint hash. At execution, source bytes are checked again, copied into the new root with identical hashes, and recorded with reuse_original_path/reused_without_prediction. Source files/manifest/STOP are not changed. The original manifest validator then sees the canonical new-root paths; predict0 is marked REUSED_BY_HASH_NO_WORKER rather than a fabricated new worker PASS.
Training prerequisite: all six original train receipts are rechecked against TRON's hashes; monitor_errors must be integer0, statusPASS/returncode0, and each embedded wrapper receipt must equal the original receipt. Original training terminal/model freezes and process-exit check are retained through terminal_inputs.
GPU selection: UUID GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36 (GPU1) only, never Track A's2/3. One root lock, one GPU lock, existing-process check and started refusal prevent a duplicate continuation. Before each prediction worker, wait at most20s for selected GPU idle, then require the original three idle polls. The wait occurs inside that guard's clock; it and all setup/poll time count toward900s per job and cumulative7200GPU-s postfit/120000GPU-s item cap. Timeout or failure stops once, no GPU switching, extra recovery or job retry. Original sampled monitor/CPU/RSS/disk/failed-peak checks remain.
Accounting: new postfit ledger = exact old postfit amount + new amount; training is added once only. Pending slots retain original900GPU-s/300CPU-s reservations. New CPU starts with180s setup allowance (covering preparation/import/synthetic overhead); child CPU and coordinator CPU use the original max-reconciliation method. The900CPU-s score and120CPU-s terminal reserve remain, inside total3600postfit CPU-s. Both the old failure and fresh usage remain auditable; no budget reset.
Barriers: freeze all adopted models; predict1–6; validate all4582 canonical entries including predict0 aliases and121original S1 test predictions; freeze combined prediction manifest/source; validate the new actual review and both barriers; then execute the unchanged one-read552-part scorer. It retains original320learning/111validation/121test targets, PPS/multiplicity/clocks/component bootstrap and original score policy. No Track A postfit or seed1 launch is attached.
Validation:15 narrow CPU synthetic checks PASS, test-body2.229610314CPU-s/2.240639487wall-s (imports excluded), plus preparation metadata/opaque-hash2.165440337CPU-s. Tests cover recovery scope, GPU1-only, preserved budgets, zero-worker stop identity, once-only accounting, six training receipts, byte-identical copy/no-overwrite, bad-hash rejection before writing,20s timeout with zero worker launches, wait+three polls charged, and monitor-error rejection. First attempt had one incorrect test expectation: seven reservations plus the selected synthetic CPU still fit3570.833960341<3600. Its failure record and source are preserved; the test fixture was corrected without changing production limits. No CUDA, scientific arrays, new predictions or score were executed by these checks.
Next: actual MeltAssist narrow delta review of resume/reuse/GPU/wait; adopt and freeze its exact receipt, then one authorized launch. Original failure evidence and training-only Track A remain untouched.
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,8 @@
{
"torch": "2.5.1+cu124",
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"checkpoint_sha256": "b9bf0d7a567ad360feb220b35868fff3ef6eb9ccb9ca7134170cf6165b81ace7",
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
"dtype": "FP32",
"autocast": false
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,11 @@
{
"status": "COMPLETE",
"parts": 343,
"entries": 686,
"targets_read": false,
"new_fits": 0,
"wall_seconds": 67.36756096500903,
"cpu_seconds": 67.35036674499999,
"peak_allocated_bytes": 229690368,
"peak_reserved_bytes": 262144000
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,16 @@
{
"pid": 1291483,
"unix": 1794978800.0724819,
"training_gpu_seconds_once": 71192.071785253,
"postfit_gpu_cap": 7200,
"item_gpu_cap": 120000,
"review_sha256": "75bd9ff11abab4420088c378a74cbf750662602706de9c2cc9c08f7c059952a2",
"cpu_start_counters": {
"self_process_time": 2.629759017,
"children_user": 0.0,
"children_system": 0.0
},
"cpu_setup_allowance": 180,
"new_fits": 0,
"new_decodes": 0
}
@@ -0,0 +1,56 @@
{
"status": "PASS",
"unix": 1794978798.618631,
"source": "Claude1794978153 existing-entry handoff exception; original native route Transport closed",
"rows": [
{
"job": 0,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/0/guards/train.receipt.json",
"sha256": "b89676c572a09b24c51fd81b390aa2ee39a3e31a8a1d19f60a31f510cb054b33",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
},
{
"job": 1,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/1/guards/train.receipt.json",
"sha256": "f7f3cebcf27a3699175ebaa6ce83f240ec511af3357cebe122bf70fd041291f8",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
},
{
"job": 2,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/2/guards/train.receipt.json",
"sha256": "a701869fecb71665efbd995128085cc638d0a2894c4a6d7bcc0818cad9a1197e",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
},
{
"job": 3,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/3/guards/train.receipt.json",
"sha256": "2e57ad1d1aa4f72c6da7fa6b01eaee4357d7966e9ba9322cf4c1a4f2a323b7de",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
},
{
"job": 4,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/4/guards/train.receipt.json",
"sha256": "2b782305a2f486d9a30421a91ea3fc6cb29c75347b4e877fbf5314493a2a6509",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
},
{
"job": 5,
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/5/guards/train.receipt.json",
"sha256": "202fd35cbdace0abbcb13f762ef36d72b473672c29e630a020d1fed7ca745a90",
"monitor_errors": 0,
"status": "PASS",
"returncode": 0
}
],
"source_edits": false
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,62 @@
{
"reviewed_commit": "8a3962b4c1267a7f021e392098098fb7ff20f562",
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
"implementation_proof_sha256": "a8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f",
"comparison_contract_sha256": "332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "fa7de3f98b2941dd1b7b78da5b06be406927a91fb40240fd746bec86a42c18fa",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "0977ce59a628f42a4870adc80c34f2d7ec810ca1e4eb9db2f1a119e096758b09",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "ae3f0849b8cf0abba75d3a7fd7b14d756234a4c094c23e446c5aaa61b75d7c79",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "b05ee93cfac1f032c84b5a0e33e6d052a4f3b507ad6c717aa6d42456ae4bc512",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "d8e975b0a5362be7ff00fbf554f2b438b54af1b6b95f8993e7c2beae77f2c8f2",
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "5465b34be5578cc62a7afc67e3cee3cd489ebb75ce6622a3fa320d1b81196c57",
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "7bfc81d820a1be7d40b97b2c0b65a616812cb7615b99d80c4ac589788282c291",
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "5e7dbf97604b9ea9646113a9847e7232020e5f9248fad388177c32ae2a7117da",
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "1a695477ee6605d75e7e991205e3e6a49a0907a7733491af8d2771c7bc6a5c23",
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "92bca4d348f31246c2e931949e2ca38103881b1b784a621a42044db56a460026",
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "8e9f781cc4975beff245d991a9b886e0d8e7bf6d66c0ff0337d76dccaddda8e1",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "987c26135c9ec3f4411925d16184e091c874a79a3b8935e72e39a3cbdb80ebed",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "1bb63d4ccf111280dd1a34cec7aeaa28026acc319a27364eb0ff57efee4d1e11",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "f203400f33bb345dee46eac7599f24b8938084124d7b6df1b3bc171537611e6e",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "e17bdf3e123a27c9187915945c17da31f24af593760a71ba91e321e5eb998a06",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "076a1a07b685a8fe5208954fdc6365cf5176f0efaf785697fdfbdc5614074569",
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "898f114a5bbc277b347ced539b36017f645e72d6b507b57f027334d6c0acb162",
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "121cf1998f03c26a841168caaa6277de1ca29948686ab85845aaab0860ec9f21",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "87fea9a1fc7b5362cb97ee0236cde9915fc65df152d1579ffb55194df1c1bd9a",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "1644feb22a276a43940453ee2babe036348c576faba99b7d407c3abc0a8bd93f",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "b5d6579a01c04e4bb4869878dcb59dc2c3d332807cf673574a580bcaf90cc816",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "1627eadaecfe58a64748650cc6100085827df227a66ffb230ad0229c7b49e79e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17"
}
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
"proof_sha256": "a8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f"
}
@@ -0,0 +1,143 @@
{
"commit": "8a3962b4c1267a7f021e392098098fb7ff20f562",
"files": {
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json": "99c9aec2149e0056712617e87c0b5c1402da2c28d249c8a4c86cdb13c015c387",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json": "4f95177490b0d437a48cfae248439ab6cbe4ea74a9067b97f83b40b561f8c771",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log": "611caf0266b345346dc775cfc7891055f4c0d3acc2f8fe01a37553f81291b3f7",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json": "8e40fe4d5c9e329bba89541d1187ef0dd243ca390c5685c4b7e097c70fba48bf",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "fa7de3f98b2941dd1b7b78da5b06be406927a91fb40240fd746bec86a42c18fa",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "0977ce59a628f42a4870adc80c34f2d7ec810ca1e4eb9db2f1a119e096758b09",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "ae3f0849b8cf0abba75d3a7fd7b14d756234a4c094c23e446c5aaa61b75d7c79",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "b05ee93cfac1f032c84b5a0e33e6d052a4f3b507ad6c717aa6d42456ae4bc512",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "d8e975b0a5362be7ff00fbf554f2b438b54af1b6b95f8993e7c2beae77f2c8f2",
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "5465b34be5578cc62a7afc67e3cee3cd489ebb75ce6622a3fa320d1b81196c57",
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "7bfc81d820a1be7d40b97b2c0b65a616812cb7615b99d80c4ac589788282c291",
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "5e7dbf97604b9ea9646113a9847e7232020e5f9248fad388177c32ae2a7117da",
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "1a695477ee6605d75e7e991205e3e6a49a0907a7733491af8d2771c7bc6a5c23",
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "92bca4d348f31246c2e931949e2ca38103881b1b784a621a42044db56a460026",
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "8e9f781cc4975beff245d991a9b886e0d8e7bf6d66c0ff0337d76dccaddda8e1",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "987c26135c9ec3f4411925d16184e091c874a79a3b8935e72e39a3cbdb80ebed",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "1bb63d4ccf111280dd1a34cec7aeaa28026acc319a27364eb0ff57efee4d1e11",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "f203400f33bb345dee46eac7599f24b8938084124d7b6df1b3bc171537611e6e",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "e17bdf3e123a27c9187915945c17da31f24af593760a71ba91e321e5eb998a06",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "076a1a07b685a8fe5208954fdc6365cf5176f0efaf785697fdfbdc5614074569",
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "898f114a5bbc277b347ced539b36017f645e72d6b507b57f027334d6c0acb162",
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "121cf1998f03c26a841168caaa6277de1ca29948686ab85845aaab0860ec9f21",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "87fea9a1fc7b5362cb97ee0236cde9915fc65df152d1579ffb55194df1c1bd9a",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "1644feb22a276a43940453ee2babe036348c576faba99b7d407c3abc0a8bd93f",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "b5d6579a01c04e4bb4869878dcb59dc2c3d332807cf673574a580bcaf90cc816",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "1627eadaecfe58a64748650cc6100085827df227a66ffb230ad0229c7b49e79e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/complete.json": "8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/started.json": "ee41d9acf11df51955d24c9482d66ba11118d3ac8f3c3c5ac52fce5cb1c70679",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/resource-progress.json": "bb2c8acf9943969ef41fd89963ffa801241d90488f78fbafa13374d967dc7db0",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json": "cbf5d113b582428171852d9b46a5371ecc7a49dac070b107bd99f82a9404e8aa",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json": "f721eacf30a0f421c8bae814be119fde6eab103b346e516add3b859962b611ff",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json": "0e36bce309bbd73d6046f81b2736881e74e7654367e399ef880cdf6c9a5ade60",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json": "6d103bf9fcdff8c73faeafbc5203568e34c1ccf6a2f9c34f955fa379ec8d5530",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json": "45ea6f53df849ecc6f92c377b3a6ef6a7a0e8eda886a7100cdd289637a70880e"
},
"git_paths": {
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json": "home-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json": "home-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json": "artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log": "artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json": "artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "tools/test_phase3_item1_postfit.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "tools/phase3_item1_postfit_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "tools/phase3_item1_postfit_data.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "tools/test_phase2_gate_outcome_postfit.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "tools/phase2_gate_outcome_postfit.py",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "tools/r4bc_guard.py",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "tools/r4bc_common.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "tools/s2_recovery_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "tools/phase2_b1213_features.py",
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "tools/phase2_b12_math.py",
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "tools/o1_v2.py",
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "tools/o1_partition.py",
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "tools/o1_prepare.py",
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "tools/gre24_features.py",
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "tools/boundary_distance.py",
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "tools/audit_samples.py",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "tools/gre24_features_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "tools/phase2_gate_outcome_statistics.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "tools/phase3_item1_postfit_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "tools/phase3_item1_postfit_statistics.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "tools/phase3_item1_postfit_common.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "tools/prepare_phase3_item1_postfit.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "tools/phase3_item1_postfit.py",
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "docs/phase3-item1-postfit-2026-11-18.md",
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "tools/revise_phase3_item1_postfit_monitor.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "tools/phase3_item1_resume.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "tools/phase3_item1_resume_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "tools/phase3_item1_resume_guard.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "tools/prepare_phase3_item1_resume.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "tools/test_phase3_item1_resume.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/complete.json": "home-runs/phase3-item1-postfit-2026-11-18/complete.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/started.json": "home-runs/phase3-item1-postfit-2026-11-18/started.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/resource-progress.json": "home-runs/phase3-item1-postfit-2026-11-18/resource-progress.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json": "home-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,17 @@
{
"status": "PREPARED_PENDING_NARROW_DELTA_REVIEW",
"metadata_and_hash_cpu_seconds": 2.165440337,
"reuse_files": 686,
"reuse_bytes": 206411454,
"original_models_exact_match": true,
"six_monitor_errors_all_integer_zero": true,
"old_sources_unchanged": true,
"original_complete_sha256": "8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8",
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
"no_model_deserialization": true,
"no_prediction_array_decode": true,
"target_values_read": false,
"new_prediction": false,
"new_score": false,
"launched": false
}
+141
View File
@@ -0,0 +1,141 @@
"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
import fcntl
import os
from pathlib import Path
import resource
import shutil
import sys
import time
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item1_postfit_common import read, manifest_identity, require_budget
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
from phase3_item1_resume_guard import run_prediction
from r4bc_guard import supervise
def install_reuse(root, entries):
result = []
for e in entries:
original = checked(e['path'], e['sha256'])
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
raise FileExistsError('No duplicate prediction adoption')
shutil.copyfile(original, target)
checked(target, e['sha256'])
result.append(dict(e, path=str(target), reuse_original_path=str(original),
reused_without_prediction=True))
out = root / 'jobs/0/prediction-records'
out.mkdir(parents=True, exist_ok=False)
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
return result
def run(root):
gpu_clock = time.monotonic()
start = time.process_time()
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, pilot = config(root)
validate_review(root)
adopted = adoption(c, tc)
lock = (root / 'run.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'started.json').exists():
raise FileExistsError('One labelled recovery, no repeat launch')
for p in psutil.process_iter(['pid', 'cmdline']):
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
raise RuntimeError('Existing original or resumed postfit')
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
new_gpu = 0.
charged_children = 0.
records = []
prediction_start = None
def costs():
u = resource.getrusage(resource.RUSAGE_CHILDREN)
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
return cumulative_cost(c, new_gpu, new_cpu)
def budget(remaining):
d = costs()
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
return d
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
try:
budget(6)
entries = install_reuse(root, reuse_entries(c, tc))
save(root / 'models-adopted.json', adopted)
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
Path(c['original_root']) / 'models-adopted.json',
*[Path(m['checkpoint']) for m in adopted['models']],
*[Path(m['standardization']) for m in adopted['models']]])
for index in c['resume_indices']:
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(7 - index)
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
prediction_start = time.monotonic()
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
prediction_start = None
new_gpu += cost
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
d = costs()
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
raise RuntimeError('Recovery prediction/resource guard failure')
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
manifest_identity(entries, c, tc, pilot)
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
root / 'predictions-manifest.json', root / 'resource-progress.json',
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(0)
validate_review(root)
verify(root, 'models')
verify(root, 'predictions')
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
charged_children += g['cpu_seconds']
records.append(dict(mode='score', guard=g))
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
raise RuntimeError('Score/resource failure')
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
future_phase3_started=False, no_retry=True))
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
final = costs()
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
if not within:
raise RuntimeError('Final cumulative resource cap')
except BaseException as exc:
if prediction_start is not None:
new_gpu += 60 + time.monotonic() - prediction_start
if (root / 'complete.json').exists():
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
new_fits=0, no_retry=True, recovery_attempt=1))
raise
if __name__ == '__main__':
run(Path(sys.argv[1]))
+120
View File
@@ -0,0 +1,120 @@
"""One labelled infrastructure continuation; original scientific modules unchanged."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_item1_postfit_common import config as original_config, read, terminal_inputs
ALLOWED_GPUS = {'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'}
def validate_scope(c, old):
if c['recovery_approval'] != '1794980643.205679' or c['resume_indices'] != list(range(1, 7)):
raise PermissionError('Exactly one recovery for jobs1 through6')
if (c['gpu_uuid'] not in ALLOWED_GPUS or c['recovery_attempt'] != 1
or c['idle_wait_seconds'] != 20 or c['idle_wait_approval'] != '1794981566.532619'):
raise PermissionError('Reserved Track A GPU or extra recovery')
for key in ('jobs', 'rows', 'training_root', 'training_contract', 'comparison_contract_sha256',
'postfit_gpu_cap', 'postfit_cpu_cap', 'item_gpu_cap', 'new_fits', 'new_decodes'):
if c[key] != old[key]:
raise ValueError('Original science/budget differs: ' + key)
if c['postfit_root'] == old['postfit_root']:
raise ValueError('Original stopped root must remain untouched')
for p, h in old['code_sha256'].items():
if c['code_sha256'].get(p) != h:
raise ValueError('Original pinned source changed')
def validate_stop(done):
records = done['records']
if done['status'] != 'STOPPED' or len(records) != 2 or [x['index'] for x in records] != [0, 1]:
raise ValueError('Unexpected original stop scope')
first, failed = [x['guard'] for x in records]
if first['status'] != 'PASS' or first['monitor_errors'] or not first['worker'] or first['worker']['status'] != 'PASS' or first['worker'].get('monitor_errors') != 0:
raise ValueError('Predict0 was not a clean PASS')
if failed['status'] != 'STOPPED' or failed['worker'] is not None or 'RuntimeError: SELECTED_GPU_BUSY_NO_RETRY' not in failed['reasons']:
raise ValueError('Not the authorized zero-worker infrastructure stop')
for key in ('prediction_gpu_seconds', 'postfit_cpu_seconds', 'training_gpu_seconds_once'):
if not math.isfinite(done[key]) or done[key] < 0:
raise ValueError('Invalid original ledger')
def six_receipts(c):
source = Path(c['original_root']) / 'tron-six-training-receipts-check.json'
check = read(source)
if check['status'] != 'PASS' or [r['job'] for r in check['rows']] != list(range(6)):
raise ValueError('Original six-receipt check missing')
for row in check['rows']:
p = Path(c['training_root']) / 'jobs' / str(row['job']) / 'guards/train.receipt.json'
if str(p) != row['path']:
raise ValueError('Original training receipt path mismatch')
receipt = read(checked(p, row['sha256']))
if type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0:
raise ValueError('Training monitoring receipt not clean')
guard = read(p.parent.parent / 'complete.json')
if guard['worker'] != receipt:
raise ValueError('Embedded training receipt differs')
return check
def config(root):
c, tc, pilot = original_config(root)
original = Path(c['original_root'])
old, _, _ = original_config(original)
validate_scope(c, old)
done = read(checked(original / 'complete.json', c['original_complete_sha256']))
validate_stop(done)
if any((original / n).exists() for n in ('score', 'predictions-manifest.json')):
raise ValueError('Original scoring or full prediction barrier already exists')
for k, name in [('prior_postfit_gpu', 'prediction_gpu_seconds'), ('prior_postfit_cpu', 'postfit_cpu_seconds'),
('training_gpu_once', 'training_gpu_seconds_once')]:
if c[k] != done[name]:
raise ValueError('Original cost must be retained exactly once')
six_receipts(c)
return c, tc, pilot
def adoption(c, tc):
current = terminal_inputs(c, tc)
original = Path(c['original_root'])
verify(original, 'models')
old = read(original / 'models-adopted.json')
if old != current or current['training_gpu_seconds_once'] != c['training_gpu_once']:
raise ValueError('Predict0 and resumed jobs must use identical frozen models')
return current
def reuse_entries(c, tc, *, verify_bytes=True):
original = Path(c['original_root'])
manifest = read(checked(original / 'jobs/0/prediction-records/manifest.json', c['predict0_manifest_sha256']))
result = read(original / 'jobs/0/prediction-records/result.json')
if result['status'] != 'COMPLETE' or result['targets_read'] or result['parts'] != c['jobs'][0]['expected_parts']:
raise ValueError('Predict0 result mismatch')
entries = manifest['entries']
f = tc['pools']['25']
domains = {'learning': 'own_learning', 'validation': 'common_validation', 'test': 'common_test'}
expected = {(arm, case): domain for role, domain in domains.items() for case in f[role]
for arm in ('epoch25', 'c25F')}
if len(entries) != len(expected) or {(e['arm'], e['case']) for e in entries} != set(expected):
raise ValueError('Complete predict0 model and clock coverage required')
if manifest['targets_read'] or result['entries'] != len(entries):
raise ValueError('Predict0 targets/entry count')
for e in entries:
p = original / 'predictions' / e['arm'] / (e['case'] + '.npz')
if (e['path'] != str(p) or e['pool'] != 25 or e['domain'] != expected[(e['arm'], e['case'])]
or e['component_id'] != c['rows'][e['case']]['component_id']
or e['origin'] != 'ITEM1_FROZEN_ENDPOINT_OR_CLOCK'):
raise ValueError('Predict0 original identity')
if verify_bytes:
checked(p, e['sha256'])
return entries
def cumulative_cost(c, new_gpu, new_cpu):
if any(not math.isfinite(v) or v < 0 for v in [new_gpu, new_cpu]):
raise ValueError('Invalid new cost ledger')
gpu = c['prior_postfit_gpu'] + new_gpu
return dict(prediction_gpu_seconds=gpu, postfit_cpu_seconds=c['prior_postfit_cpu'] + new_cpu,
training_gpu_seconds_once=c['training_gpu_once'],
cumulative_item_gpu_seconds=c['training_gpu_once'] + gpu)
+85
View File
@@ -0,0 +1,85 @@
"""Recovery-only guard: bounded pre-worker idle wait inside original900s cap."""
import json,os,signal,sys,threading,time
import psutil
from phase2_h1_worker_v2 import save
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
out=list(reasons)
def add(s):
if s not in out:out.append(s)
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
return out
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
def elapsed():return 60+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase3_item1_postfit_worker.py','predict',str(queue),str(fold)]
def kill():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
def scan():
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
if rss>16*2**30:reason('TREE_RSS_LIMIT')
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
for x in requested:reason(x)
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
if reasons:kill()
done.wait(.5)
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
try:
wait_start=time.monotonic();waiting=[]
while True:
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
if requested:raise RuntimeError('Signal received during idle wait')
duration=time.monotonic()-wait_start
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
if mem<=128*2**20 and util==0:break
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
for i in range(3):
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
if i<2:time.sleep(2)
save(root/'prediction-idle-polls.json',samples)
if requested:raise RuntimeError('Signal received during setup')
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=900-elapsed()-10
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
for s,h in old.items():signal.signal(s,h)
result=None;p=root/'prediction-records/result.json'
try:
if p.exists():result=json.loads(p.read_text())
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_item1_gpu_seconds=prior_gpu+duration,no_retry=True)
save(root/'prediction-guard.json',out);return out
+69
View File
@@ -0,0 +1,69 @@
"""Metadata and opaque-hash preparation; no new prediction or target access."""
from pathlib import Path
import time
from phase2_h1_worker_v2 import checked, save, sha
from phase2_gate_outcome_review import validate_review
from phase2_home_freeze_io import verify
from phase3_item1_postfit_common import config as old_config, read
from phase3_item1_resume_common import validate_scope, validate_stop, six_receipts, adoption, reuse_entries
from track_a_h2_prepare import local_code_closure
B = Path(__file__).resolve().parents[1]
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
R = O.parent / 'phase3-item1-postfit-recovery-2026-11-18'
def prepare():
start = time.process_time()
if R.exists():
raise FileExistsError('Existing recovery entry: inspect before any further action')
old, tc, pilot = old_config(O)
validate_review(O)
done = read(O / 'complete.json')
validate_stop(done)
sources = dict(old['source_sha256'])
for name in ['contract.json', 'complete.json', 'started.json', 'resource-progress.json',
'tron-six-training-receipts-check.json', 'models-adopted.json', 'models-git-freeze.json',
'review.json', 'implementation-git-freeze.json',
'jobs/0/prediction-guard.json', 'jobs/0/prediction-records/manifest.json',
'jobs/0/prediction-records/result.json', 'jobs/0/prediction-records/environment.json']:
sources[str(O / name)] = sha(O / name)
codes = dict(old['code_sha256'])
additional = local_code_closure([B / 'tools' / n for n in ['phase3_item1_resume.py',
'phase3_item1_resume_common.py', 'phase3_item1_resume_guard.py', 'prepare_phase3_item1_resume.py',
'test_phase3_item1_resume.py']])
for p, h in additional.items():
if p in codes and codes[p] != h:
raise ValueError('Old pinned code changed')
codes[p] = h
c = dict(old, postfit_root=str(R), code_sha256=codes, source_sha256=sources,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
original_root=str(O), original_complete_sha256=sha(O / 'complete.json'),
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1, 7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
predict0_manifest_sha256=sha(O / 'jobs/0/prediction-records/manifest.json'),
prior_postfit_gpu=done['prediction_gpu_seconds'], prior_postfit_cpu=done['postfit_cpu_seconds'],
training_gpu_once=done['training_gpu_seconds_once'],
engineering_revision='Single labelled predict1..6 continuation; immutable predict0 by hash',
same_independent_review=False)
validate_scope(c, old)
six_receipts(c)
adopted = adoption(c, tc)
entries = reuse_entries(c, tc)
env = read(O / 'jobs/0/prediction-records/environment.json')
if env['checkpoint_sha256'] != adopted['models'][0]['checkpoint_sha256']:
raise ValueError('Predict0 checkpoint differs')
R.mkdir()
save(R / 'contract.json', c)
save(R / 'preparation-receipt.json', dict(status='PREPARED_PENDING_NARROW_DELTA_REVIEW',
metadata_and_hash_cpu_seconds=time.process_time() - start, reuse_files=len(entries),
reuse_bytes=sum(Path(e['path']).stat().st_size for e in entries), original_models_exact_match=True,
six_monitor_errors_all_integer_zero=True, old_sources_unchanged=True,
original_complete_sha256=c['original_complete_sha256'], contract_sha256=sha(R / 'contract.json'),
no_model_deserialization=True, no_prediction_array_decode=True, target_values_read=False,
new_prediction=False, new_score=False, launched=False))
print(read(R / 'preparation-receipt.json'))
if __name__ == '__main__':
prepare()
+125
View File
@@ -0,0 +1,125 @@
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
import copy
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save, sha
from phase3_item1_postfit_common import read, require_budget
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
from phase3_item1_resume import install_reuse
import phase3_item1_resume_guard as guard
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.old = read(O / 'contract.json')
self.done = read(O / 'complete.json')
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
training_gpu_once=self.done['training_gpu_seconds_once'])
def test_scope_and_actual_original_stop(self):
validate_scope(self.c, self.old)
validate_stop(self.done)
def test_no_predict0_rerun(self):
self.c['resume_indices'] = list(range(7))
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_gpu1_only(self):
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
self.c['gpu_uuid'] = uuid
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_no_budget_change(self):
self.c['postfit_gpu_cap'] = 7201
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
def test_failed_job_worker_must_be_null(self):
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
with self.assertRaises(ValueError): validate_stop(self.done)
def test_predict0_must_be_pass(self):
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
with self.assertRaises(ValueError): validate_stop(self.done)
def test_old_cost_once_not_per_job(self):
a = cumulative_cost(self.c, 100., 200.)
b = cumulative_cost(self.c, 300., 500.)
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
def test_old_cost_limits_remaining_reserve(self):
d = cumulative_cost(self.c,60,240)
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
with self.assertRaises(RuntimeError):
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
def test_original_six_receipts_currently_clean(self):
self.assertEqual(len(six_receipts(self.c)['rows']),6)
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
with self.assertRaises(FileExistsError):install_reuse(root,[e])
def test_reuse_bad_hash_rejected_before_write(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
with self.assertRaises(ValueError):
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
self.assertEqual(list(root.iterdir()),[])
def guarded(self, mode):
timer=[0.];calls=[0]
def sleep(seconds): timer[0]+=seconds
def probe(uuid):
calls[0]+=1
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
return 0,0
class Thread:
def __init__(self,*a,**k):pass
def start(self):pass
def join(self,**k):pass
def is_alive(self):return False
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
def supervisor(*a,**k):
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
result=guard.run_prediction(root,c,{},1,0,300)
return result,sup.call_count,read(out/'recovery-idle-wait.json')
def test_busy_then_idle_counts_wait_and_three_polls(self):
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
self.assertEqual(r['gpu_seconds_charged'],65.)
self.assertEqual(len(w['samples']),3)
def test_busy_timeout_stops_without_worker(self):
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
self.assertEqual(r['gpu_seconds_charged'],80.)
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
def test_supervisor_monitor_error_still_rejected(self):
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
def test_wait_limit_cannot_be_relaxed(self):
self.c['idle_wait_seconds']=21
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
if __name__ == '__main__':unittest.main()