Prepare one labelled item1 postfit recovery with bounded idle wait
This commit is contained in:
1 parent
4f646232f4
commit
581208098f
24 files changed
+83789
No files matched your search
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 15,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"cpu_seconds": 2.229610314,
|
||||
"wall_seconds": 2.240639487048611,
|
||||
"source_sha256": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
|
||||
"GPU_used": false,
|
||||
"real_scientific_arrays_read": false,
|
||||
"attempt01_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
|
||||
import copy
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase3_item1_postfit_common import read, require_budget
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
|
||||
from phase3_item1_resume import install_reuse
|
||||
import phase3_item1_resume_guard as guard
|
||||
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
|
||||
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = read(O / 'contract.json')
|
||||
self.done = read(O / 'complete.json')
|
||||
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
|
||||
training_gpu_once=self.done['training_gpu_seconds_once'])
|
||||
|
||||
def test_scope_and_actual_original_stop(self):
|
||||
validate_scope(self.c, self.old)
|
||||
validate_stop(self.done)
|
||||
|
||||
def test_no_predict0_rerun(self):
|
||||
self.c['resume_indices'] = list(range(7))
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_gpu1_only(self):
|
||||
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
|
||||
self.c['gpu_uuid'] = uuid
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_no_budget_change(self):
|
||||
self.c['postfit_gpu_cap'] = 7201
|
||||
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_failed_job_worker_must_be_null(self):
|
||||
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_predict0_must_be_pass(self):
|
||||
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_old_cost_once_not_per_job(self):
|
||||
a = cumulative_cost(self.c, 100., 200.)
|
||||
b = cumulative_cost(self.c, 300., 500.)
|
||||
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
|
||||
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
|
||||
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
|
||||
|
||||
def test_old_cost_limits_remaining_reserve(self):
|
||||
d = cumulative_cost(self.c,60,180)
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
|
||||
with self.assertRaises(RuntimeError):
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
|
||||
|
||||
def test_original_six_receipts_currently_clean(self):
|
||||
self.assertEqual(len(six_receipts(self.c)['rows']),6)
|
||||
|
||||
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
|
||||
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
|
||||
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
|
||||
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
|
||||
with self.assertRaises(FileExistsError):install_reuse(root,[e])
|
||||
|
||||
def test_reuse_bad_hash_rejected_before_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
|
||||
with self.assertRaises(ValueError):
|
||||
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
|
||||
self.assertEqual(list(root.iterdir()),[])
|
||||
|
||||
def guarded(self, mode):
|
||||
timer=[0.];calls=[0]
|
||||
def sleep(seconds): timer[0]+=seconds
|
||||
def probe(uuid):
|
||||
calls[0]+=1
|
||||
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
|
||||
return 0,0
|
||||
class Thread:
|
||||
def __init__(self,*a,**k):pass
|
||||
def start(self):pass
|
||||
def join(self,**k):pass
|
||||
def is_alive(self):return False
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
|
||||
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
|
||||
def supervisor(*a,**k):
|
||||
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
|
||||
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
|
||||
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
|
||||
result=guard.run_prediction(root,c,{},1,0,300)
|
||||
return result,sup.call_count,read(out/'recovery-idle-wait.json')
|
||||
|
||||
def test_busy_then_idle_counts_wait_and_three_polls(self):
|
||||
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
|
||||
self.assertEqual(r['gpu_seconds_charged'],65.)
|
||||
self.assertEqual(len(w['samples']),3)
|
||||
|
||||
def test_busy_timeout_stops_without_worker(self):
|
||||
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
|
||||
self.assertEqual(r['gpu_seconds_charged'],80.)
|
||||
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
|
||||
|
||||
def test_supervisor_monitor_error_still_rejected(self):
|
||||
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
|
||||
|
||||
def test_wait_limit_cannot_be_relaxed(self):
|
||||
self.c['idle_wait_seconds']=21
|
||||
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"status": "FAIL",
|
||||
"tests": 15,
|
||||
"failures": 1,
|
||||
"errors": 0,
|
||||
"unittest_wall_seconds": 1.478,
|
||||
"test_source_sha256": "0835350eae48f3766d8bf34caa58eefe2d0670e5ebea092a47f7df744877c4ea",
|
||||
"failure": "test_old_cost_limits_remaining_reserve expected7 remaining to exceed cap at newCPU180, but total3570.833960341 remains below3600. Incorrect synthetic assertion; no production budget change.",
|
||||
"real_scientific_execution": false
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
test_busy_then_idle_counts_wait_and_three_polls (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_busy_timeout_stops_without_worker (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_failed_job_worker_must_be_null (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_gpu1_only (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_no_budget_change (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_no_predict0_rerun (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_old_cost_limits_remaining_reserve (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_old_cost_once_not_per_job (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_original_six_receipts_currently_clean (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_predict0_must_be_pass (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_reuse_bad_hash_rejected_before_write (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_reuse_copy_preserves_original_and_rejects_duplicate (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_scope_and_actual_original_stop (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_supervisor_monitor_error_still_rejected (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_wait_limit_cannot_be_relaxed (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 15 tests in 2.240s
|
||||
|
||||
OK
|
||||
@@ -1,3 +1,11 @@
|
||||
## 最新 item1 后处理最小恢复已固定待窄审(2026-11-18)
|
||||
|
||||
已实读Claude0643/9347及1566:原postfit STOP保留,predict0的686文件逐字hash复用,新根phase3-item1-postfit-recovery-2026-11-18只predict1–6;GPU1only、每job20s等待后原三idle polls,全部在900s及总账内,无换卡/重试。实现8a3962b4/contract7e991ea1/proofa8e671ff,comparison332f2515不变,54pins含全部原47不变。15CPU合成PASS,首测试断言失败/源已保留;尚无review/started/新预测。旧GPU261.150194630/CPU270.833960341/训练71192.071785253各计一次。详见docs/phase3-item1-postfit-recovery-2026-11-18.md;须Assist窄delta后执行,旧PASS不作新审阅。Track A原队列不改、2/3保留。
|
||||
|
||||
## 最新 Track A 首两个epoch成本门PASS(2026-11-18)
|
||||
|
||||
原唯一1262364继续,job0/1 worker1262432/1262422已进入epoch2;完整首epoch61.848708608s/59.255394293s,逐job门均PASS。最新总投影130716.145896GPU-s/162000,含未启动2/3/4整帽和7200s postfit预留,不是全5测量。source6701e4d6/contractbe4ac1f6不变,无重启/改源码。审阅收讫+实际启动+两门证据已远端固定4f646232,报告和原始first-two-epoch-audit.json保留。队列只checkpoint后停,test/predict/score需后续原pre-score审阅。
|
||||
|
||||
## 最新 Track A H2 唯一训练已实际启动(2026-11-18 12:28)
|
||||
|
||||
Assist1794975952原件1fa923f8已逐字接收/固定b7879eec并validate PASS。唯一service meltmind-track-a-h2-20261118.service/coord1262364,root /home/frank14f/MeltMind-runs/track-a-h2-2026-11-18;job0 guard1262373/worker1262432 GPU2,job1 guard1262374/worker1262422 GPU3,首epoch真实51/311和58/304快照,三idle polls各PASS。source6701e4d6/contractbe4ac1f6不变,23pins PASS。job2/3/4 pending,162000GPU-s含7200postfit预留,各job首epoch成本门自动执行,尚未成本PASS;禁止重复启动/改运行源。item1旧worker1135393仍GPU1(56503/64000),GPU0保留其postfit。队列只freeze模型后结束,test/predict/score仍须原独立pre-score审阅,训练PASS不得替代。实际证据actual-start-audit.json。下方未启动/待审仅历史。
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Item1 postfit: one labelled infrastructure continuation
|
||||
|
||||
Prepared and locally frozen; narrow delta review pending, no new prediction or scoring started. Claude1794979347/1794980643 authorize reuse of predict0 and exactly predict1–6; Claude1794981566 fixes GPU1 only and the bounded20s idle wait. The original STOP is infrastructure-labelled with cause undiagnosed, not a scientific failure or a proven utilization-sampling issue.
|
||||
|
||||
Original root `/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18` remains untouched. Its complete SHA8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8 records predict0 PASS, predict1 stopped before any worker (`worker=null`), GPU261.1501946300268s and CPU270.833960341s. Training71192.071785253GPU-s remains a separate once-only amount. Original started, resource-progress, six-receipt audit and raw guards are retained.
|
||||
|
||||
New root `/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18`. Implementation8a3962b4c1267a7f021e392098098fb7ff20f562; contract7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622; proofa8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f. Original comparison332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9 stays byte-identical. expected-review-binding.json includes54 code pins; all original47 pins remain identical. This new delta does not reuse old PASS75bd9ff1 as its own verdict.
|
||||
|
||||
New files are `phase3_item1_resume.py`, `phase3_item1_resume_common.py`, `phase3_item1_resume_guard.py`, `prepare_phase3_item1_resume.py`, and `test_phase3_item1_resume.py`. The original prediction worker, feature loader, score, statistics and comparison functions are imported/run unchanged. The independent coordinator uses the new root and original seven endpoint metadata, but dispatches only indices1..6. No training, model-selection or field-decoding step is added.
|
||||
|
||||
Reuse: all686 predict0 files (206411454bytes) have their original manifest hashes checked as opaque bytes; no arrays or model values were decoded during preparation. The old manifest has complete epoch25 and c25F coverage over343parts. Model adoption is independently matched to the old adopted-model manifest and predict0 environment checkpoint hash. At execution, source bytes are checked again, copied into the new root with identical hashes, and recorded with reuse_original_path/reused_without_prediction. Source files/manifest/STOP are not changed. The original manifest validator then sees the canonical new-root paths; predict0 is marked REUSED_BY_HASH_NO_WORKER rather than a fabricated new worker PASS.
|
||||
|
||||
Training prerequisite: all six original train receipts are rechecked against TRON's hashes; monitor_errors must be integer0, statusPASS/returncode0, and each embedded wrapper receipt must equal the original receipt. Original training terminal/model freezes and process-exit check are retained through terminal_inputs.
|
||||
|
||||
GPU selection: UUID GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36 (GPU1) only, never Track A's2/3. One root lock, one GPU lock, existing-process check and started refusal prevent a duplicate continuation. Before each prediction worker, wait at most20s for selected GPU idle, then require the original three idle polls. The wait occurs inside that guard's clock; it and all setup/poll time count toward900s per job and cumulative7200GPU-s postfit/120000GPU-s item cap. Timeout or failure stops once, no GPU switching, extra recovery or job retry. Original sampled monitor/CPU/RSS/disk/failed-peak checks remain.
|
||||
|
||||
Accounting: new postfit ledger = exact old postfit amount + new amount; training is added once only. Pending slots retain original900GPU-s/300CPU-s reservations. New CPU starts with180s setup allowance (covering preparation/import/synthetic overhead); child CPU and coordinator CPU use the original max-reconciliation method. The900CPU-s score and120CPU-s terminal reserve remain, inside total3600postfit CPU-s. Both the old failure and fresh usage remain auditable; no budget reset.
|
||||
|
||||
Barriers: freeze all adopted models; predict1–6; validate all4582 canonical entries including predict0 aliases and121original S1 test predictions; freeze combined prediction manifest/source; validate the new actual review and both barriers; then execute the unchanged one-read552-part scorer. It retains original320learning/111validation/121test targets, PPS/multiplicity/clocks/component bootstrap and original score policy. No Track A postfit or seed1 launch is attached.
|
||||
|
||||
Validation:15 narrow CPU synthetic checks PASS, test-body2.229610314CPU-s/2.240639487wall-s (imports excluded), plus preparation metadata/opaque-hash2.165440337CPU-s. Tests cover recovery scope, GPU1-only, preserved budgets, zero-worker stop identity, once-only accounting, six training receipts, byte-identical copy/no-overwrite, bad-hash rejection before writing,20s timeout with zero worker launches, wait+three polls charged, and monitor-error rejection. First attempt had one incorrect test expectation: seven reservations plus the selected synthetic CPU still fit3570.833960341<3600. Its failure record and source are preserved; the test fixture was corrected without changing production limits. No CUDA, scientific arrays, new predictions or score were executed by these checks.
|
||||
|
||||
Next: actual MeltAssist narrow delta review of resume/reuse/GPU/wait; adopt and freeze its exact receipt, then one authorized launch. Original failure evidence and training-only Track A remain untouched.
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"torch": "2.5.1+cu124",
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"checkpoint_sha256": "b9bf0d7a567ad360feb220b35868fff3ef6eb9ccb9ca7134170cf6165b81ace7",
|
||||
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
|
||||
"dtype": "FP32",
|
||||
"autocast": false
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "COMPLETE",
|
||||
"parts": 343,
|
||||
"entries": 686,
|
||||
"targets_read": false,
|
||||
"new_fits": 0,
|
||||
"wall_seconds": 67.36756096500903,
|
||||
"cpu_seconds": 67.35036674499999,
|
||||
"peak_allocated_bytes": 229690368,
|
||||
"peak_reserved_bytes": 262144000
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"pid": 1291483,
|
||||
"unix": 1794978800.0724819,
|
||||
"training_gpu_seconds_once": 71192.071785253,
|
||||
"postfit_gpu_cap": 7200,
|
||||
"item_gpu_cap": 120000,
|
||||
"review_sha256": "75bd9ff11abab4420088c378a74cbf750662602706de9c2cc9c08f7c059952a2",
|
||||
"cpu_start_counters": {
|
||||
"self_process_time": 2.629759017,
|
||||
"children_user": 0.0,
|
||||
"children_system": 0.0
|
||||
},
|
||||
"cpu_setup_allowance": 180,
|
||||
"new_fits": 0,
|
||||
"new_decodes": 0
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"unix": 1794978798.618631,
|
||||
"source": "Claude1794978153 existing-entry handoff exception; original native route Transport closed",
|
||||
"rows": [
|
||||
{
|
||||
"job": 0,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/0/guards/train.receipt.json",
|
||||
"sha256": "b89676c572a09b24c51fd81b390aa2ee39a3e31a8a1d19f60a31f510cb054b33",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
},
|
||||
{
|
||||
"job": 1,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/1/guards/train.receipt.json",
|
||||
"sha256": "f7f3cebcf27a3699175ebaa6ce83f240ec511af3357cebe122bf70fd041291f8",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
},
|
||||
{
|
||||
"job": 2,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/2/guards/train.receipt.json",
|
||||
"sha256": "a701869fecb71665efbd995128085cc638d0a2894c4a6d7bcc0818cad9a1197e",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
},
|
||||
{
|
||||
"job": 3,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/3/guards/train.receipt.json",
|
||||
"sha256": "2e57ad1d1aa4f72c6da7fa6b01eaee4357d7966e9ba9322cf4c1a4f2a323b7de",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
},
|
||||
{
|
||||
"job": 4,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/4/guards/train.receipt.json",
|
||||
"sha256": "2b782305a2f486d9a30421a91ea3fc6cb29c75347b4e877fbf5314493a2a6509",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
},
|
||||
{
|
||||
"job": 5,
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-data-sufficiency-v2-2026-11-18/jobs/5/guards/train.receipt.json",
|
||||
"sha256": "202fd35cbdace0abbcb13f762ef36d72b473672c29e630a020d1fed7ca745a90",
|
||||
"monitor_errors": 0,
|
||||
"status": "PASS",
|
||||
"returncode": 0
|
||||
}
|
||||
],
|
||||
"source_edits": false
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"reviewed_commit": "8a3962b4c1267a7f021e392098098fb7ff20f562",
|
||||
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
|
||||
"implementation_proof_sha256": "a8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f",
|
||||
"comparison_contract_sha256": "332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "fa7de3f98b2941dd1b7b78da5b06be406927a91fb40240fd746bec86a42c18fa",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "0977ce59a628f42a4870adc80c34f2d7ec810ca1e4eb9db2f1a119e096758b09",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "ae3f0849b8cf0abba75d3a7fd7b14d756234a4c094c23e446c5aaa61b75d7c79",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "b05ee93cfac1f032c84b5a0e33e6d052a4f3b507ad6c717aa6d42456ae4bc512",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "d8e975b0a5362be7ff00fbf554f2b438b54af1b6b95f8993e7c2beae77f2c8f2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "5465b34be5578cc62a7afc67e3cee3cd489ebb75ce6622a3fa320d1b81196c57",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "7bfc81d820a1be7d40b97b2c0b65a616812cb7615b99d80c4ac589788282c291",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "5e7dbf97604b9ea9646113a9847e7232020e5f9248fad388177c32ae2a7117da",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "1a695477ee6605d75e7e991205e3e6a49a0907a7733491af8d2771c7bc6a5c23",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "92bca4d348f31246c2e931949e2ca38103881b1b784a621a42044db56a460026",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "8e9f781cc4975beff245d991a9b886e0d8e7bf6d66c0ff0337d76dccaddda8e1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "987c26135c9ec3f4411925d16184e091c874a79a3b8935e72e39a3cbdb80ebed",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "1bb63d4ccf111280dd1a34cec7aeaa28026acc319a27364eb0ff57efee4d1e11",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "f203400f33bb345dee46eac7599f24b8938084124d7b6df1b3bc171537611e6e",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "e17bdf3e123a27c9187915945c17da31f24af593760a71ba91e321e5eb998a06",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "076a1a07b685a8fe5208954fdc6365cf5176f0efaf785697fdfbdc5614074569",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "898f114a5bbc277b347ced539b36017f645e72d6b507b57f027334d6c0acb162",
|
||||
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "121cf1998f03c26a841168caaa6277de1ca29948686ab85845aaab0860ec9f21",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "87fea9a1fc7b5362cb97ee0236cde9915fc65df152d1579ffb55194df1c1bd9a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "1644feb22a276a43940453ee2babe036348c576faba99b7d407c3abc0a8bd93f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "b5d6579a01c04e4bb4869878dcb59dc2c3d332807cf673574a580bcaf90cc816",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "1627eadaecfe58a64748650cc6100085827df227a66ffb230ad0229c7b49e79e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
|
||||
"proof_sha256": "a8e671ffa4b43998eee3ad302bb9f2a40f7ade83b3209274245c75f6fddb4d1f"
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"commit": "8a3962b4c1267a7f021e392098098fb7ff20f562",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json": "99c9aec2149e0056712617e87c0b5c1402da2c28d249c8a4c86cdb13c015c387",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json": "4f95177490b0d437a48cfae248439ab6cbe4ea74a9067b97f83b40b561f8c771",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log": "611caf0266b345346dc775cfc7891055f4c0d3acc2f8fe01a37553f81291b3f7",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json": "8e40fe4d5c9e329bba89541d1187ef0dd243ca390c5685c4b7e097c70fba48bf",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "fa7de3f98b2941dd1b7b78da5b06be406927a91fb40240fd746bec86a42c18fa",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "0977ce59a628f42a4870adc80c34f2d7ec810ca1e4eb9db2f1a119e096758b09",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "ae3f0849b8cf0abba75d3a7fd7b14d756234a4c094c23e446c5aaa61b75d7c79",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "b05ee93cfac1f032c84b5a0e33e6d052a4f3b507ad6c717aa6d42456ae4bc512",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "d8e975b0a5362be7ff00fbf554f2b438b54af1b6b95f8993e7c2beae77f2c8f2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "5465b34be5578cc62a7afc67e3cee3cd489ebb75ce6622a3fa320d1b81196c57",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "7bfc81d820a1be7d40b97b2c0b65a616812cb7615b99d80c4ac589788282c291",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "5e7dbf97604b9ea9646113a9847e7232020e5f9248fad388177c32ae2a7117da",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "1a695477ee6605d75e7e991205e3e6a49a0907a7733491af8d2771c7bc6a5c23",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "92bca4d348f31246c2e931949e2ca38103881b1b784a621a42044db56a460026",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "8e9f781cc4975beff245d991a9b886e0d8e7bf6d66c0ff0337d76dccaddda8e1",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "987c26135c9ec3f4411925d16184e091c874a79a3b8935e72e39a3cbdb80ebed",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "1bb63d4ccf111280dd1a34cec7aeaa28026acc319a27364eb0ff57efee4d1e11",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "f203400f33bb345dee46eac7599f24b8938084124d7b6df1b3bc171537611e6e",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "e17bdf3e123a27c9187915945c17da31f24af593760a71ba91e321e5eb998a06",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "076a1a07b685a8fe5208954fdc6365cf5176f0efaf785697fdfbdc5614074569",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "898f114a5bbc277b347ced539b36017f645e72d6b507b57f027334d6c0acb162",
|
||||
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "121cf1998f03c26a841168caaa6277de1ca29948686ab85845aaab0860ec9f21",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "87fea9a1fc7b5362cb97ee0236cde9915fc65df152d1579ffb55194df1c1bd9a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "1644feb22a276a43940453ee2babe036348c576faba99b7d407c3abc0a8bd93f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "b5d6579a01c04e4bb4869878dcb59dc2c3d332807cf673574a580bcaf90cc816",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "1627eadaecfe58a64748650cc6100085827df227a66ffb230ad0229c7b49e79e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/complete.json": "8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/started.json": "ee41d9acf11df51955d24c9482d66ba11118d3ac8f3c3c5ac52fce5cb1c70679",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/resource-progress.json": "bb2c8acf9943969ef41fd89963ffa801241d90488f78fbafa13374d967dc7db0",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json": "cbf5d113b582428171852d9b46a5371ecc7a49dac070b107bd99f82a9404e8aa",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json": "f721eacf30a0f421c8bae814be119fde6eab103b346e516add3b859962b611ff",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json": "0e36bce309bbd73d6046f81b2736881e74e7654367e399ef880cdf6c9a5ade60",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json": "6d103bf9fcdff8c73faeafbc5203568e34c1ccf6a2f9c34f955fa379ec8d5530",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json": "45ea6f53df849ecc6f92c377b3a6ef6a7a0e8eda886a7100cdd289637a70880e"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json": "home-runs/phase3-item1-postfit-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json": "home-runs/phase3-item1-postfit-recovery-2026-11-18/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json": "artifacts/phase3-item1-resume-acceptance-2026-11-18/acceptance.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log": "artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-02.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json": "artifacts/phase3-item1-resume-acceptance-2026-11-18/attempt-01.json",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_postfit.py": "tools/test_phase3_item1_postfit.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_guard.py": "tools/phase3_item1_postfit_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "tools/phase3_item1_postfit_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_gate_outcome_postfit.py": "tools/test_phase2_gate_outcome_postfit.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_postfit.py": "tools/phase2_gate_outcome_postfit.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "tools/r4bc_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "tools/r4bc_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "tools/s2_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b1213_features.py": "tools/phase2_b1213_features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_b12_math.py": "tools/phase2_b12_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_v2.py": "tools/o1_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_partition.py": "tools/o1_partition.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/o1_prepare.py": "tools/o1_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features.py": "tools/gre24_features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "tools/boundary_distance.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/audit_samples.py": "tools/audit_samples.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "tools/gre24_features_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_statistics.py": "tools/phase2_gate_outcome_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_worker.py": "tools/phase3_item1_postfit_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_statistics.py": "tools/phase3_item1_postfit_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_common.py": "tools/phase3_item1_postfit_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_postfit.py": "tools/prepare_phase3_item1_postfit.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit.py": "tools/phase3_item1_postfit.py",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item1-postfit-2026-11-18.md": "docs/phase3-item1-postfit-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/tools/revise_phase3_item1_postfit_monitor.py": "tools/revise_phase3_item1_postfit_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume.py": "tools/phase3_item1_resume.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_common.py": "tools/phase3_item1_resume_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_resume_guard.py": "tools/phase3_item1_resume_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1_resume.py": "tools/prepare_phase3_item1_resume.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1_resume.py": "tools/test_phase3_item1_resume.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/complete.json": "home-runs/phase3-item1-postfit-2026-11-18/complete.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/started.json": "home-runs/phase3-item1-postfit-2026-11-18/started.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/resource-progress.json": "home-runs/phase3-item1-postfit-2026-11-18/resource-progress.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json": "home-runs/phase3-item1-postfit-2026-11-18/tron-six-training-receipts-check.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-guard.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/manifest.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/result.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json": "home-runs/phase3-item1-postfit-2026-11-18/jobs/0/prediction-records/environment.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_NARROW_DELTA_REVIEW",
|
||||
"metadata_and_hash_cpu_seconds": 2.165440337,
|
||||
"reuse_files": 686,
|
||||
"reuse_bytes": 206411454,
|
||||
"original_models_exact_match": true,
|
||||
"six_monitor_errors_all_integer_zero": true,
|
||||
"old_sources_unchanged": true,
|
||||
"original_complete_sha256": "8062b900b8267d9b5a691f943beb11d533570f8cae76e3ba94e59717665204e8",
|
||||
"contract_sha256": "7e991ea1b5477f92d4bdaa99eb0629ca23d50c068a1b11a1827c189bb8c70622",
|
||||
"no_model_deserialization": true,
|
||||
"no_prediction_array_decode": true,
|
||||
"target_values_read": false,
|
||||
"new_prediction": false,
|
||||
"new_score": false,
|
||||
"launched": false
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_item1_postfit_common import read, manifest_identity, require_budget
|
||||
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
|
||||
from phase3_item1_resume_guard import run_prediction
|
||||
from r4bc_guard import supervise
|
||||
|
||||
|
||||
def install_reuse(root, entries):
|
||||
result = []
|
||||
for e in entries:
|
||||
original = checked(e['path'], e['sha256'])
|
||||
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists():
|
||||
raise FileExistsError('No duplicate prediction adoption')
|
||||
shutil.copyfile(original, target)
|
||||
checked(target, e['sha256'])
|
||||
result.append(dict(e, path=str(target), reuse_original_path=str(original),
|
||||
reused_without_prediction=True))
|
||||
out = root / 'jobs/0/prediction-records'
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
|
||||
return result
|
||||
|
||||
|
||||
def run(root):
|
||||
gpu_clock = time.monotonic()
|
||||
start = time.process_time()
|
||||
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
c, tc, pilot = config(root)
|
||||
validate_review(root)
|
||||
adopted = adoption(c, tc)
|
||||
lock = (root / 'run.lock').open('a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'started.json').exists():
|
||||
raise FileExistsError('One labelled recovery, no repeat launch')
|
||||
for p in psutil.process_iter(['pid', 'cmdline']):
|
||||
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
|
||||
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
|
||||
raise RuntimeError('Existing original or resumed postfit')
|
||||
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
|
||||
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
new_gpu = 0.
|
||||
charged_children = 0.
|
||||
records = []
|
||||
prediction_start = None
|
||||
|
||||
def costs():
|
||||
u = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
|
||||
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
|
||||
return cumulative_cost(c, new_gpu, new_cpu)
|
||||
|
||||
def budget(remaining):
|
||||
d = costs()
|
||||
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
|
||||
return d
|
||||
|
||||
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
|
||||
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
|
||||
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
|
||||
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
|
||||
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
|
||||
try:
|
||||
budget(6)
|
||||
entries = install_reuse(root, reuse_entries(c, tc))
|
||||
save(root / 'models-adopted.json', adopted)
|
||||
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
|
||||
Path(c['original_root']) / 'models-adopted.json',
|
||||
*[Path(m['checkpoint']) for m in adopted['models']],
|
||||
*[Path(m['standardization']) for m in adopted['models']]])
|
||||
for index in c['resume_indices']:
|
||||
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
||||
budget(7 - index)
|
||||
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
|
||||
prediction_start = time.monotonic()
|
||||
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
|
||||
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
|
||||
prediction_start = None
|
||||
new_gpu += cost
|
||||
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
|
||||
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
|
||||
d = costs()
|
||||
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
|
||||
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
|
||||
raise RuntimeError('Recovery prediction/resource guard failure')
|
||||
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
|
||||
manifest_identity(entries, c, tc, pilot)
|
||||
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
|
||||
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
|
||||
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
|
||||
root / 'predictions-manifest.json', root / 'resource-progress.json',
|
||||
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
|
||||
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
||||
budget(0)
|
||||
validate_review(root)
|
||||
verify(root, 'models')
|
||||
verify(root, 'predictions')
|
||||
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
|
||||
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
|
||||
charged_children += g['cpu_seconds']
|
||||
records.append(dict(mode='score', guard=g))
|
||||
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
|
||||
raise RuntimeError('Score/resource failure')
|
||||
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
|
||||
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
|
||||
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
|
||||
future_phase3_started=False, no_retry=True))
|
||||
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
|
||||
final = costs()
|
||||
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
|
||||
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
|
||||
if not within:
|
||||
raise RuntimeError('Final cumulative resource cap')
|
||||
except BaseException as exc:
|
||||
if prediction_start is not None:
|
||||
new_gpu += 60 + time.monotonic() - prediction_start
|
||||
if (root / 'complete.json').exists():
|
||||
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
|
||||
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
|
||||
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
|
||||
new_fits=0, no_retry=True, recovery_attempt=1))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,120 @@
|
||||
"""One labelled infrastructure continuation; original scientific modules unchanged."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_item1_postfit_common import config as original_config, read, terminal_inputs
|
||||
|
||||
ALLOWED_GPUS = {'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'}
|
||||
|
||||
|
||||
def validate_scope(c, old):
|
||||
if c['recovery_approval'] != '1794980643.205679' or c['resume_indices'] != list(range(1, 7)):
|
||||
raise PermissionError('Exactly one recovery for jobs1 through6')
|
||||
if (c['gpu_uuid'] not in ALLOWED_GPUS or c['recovery_attempt'] != 1
|
||||
or c['idle_wait_seconds'] != 20 or c['idle_wait_approval'] != '1794981566.532619'):
|
||||
raise PermissionError('Reserved Track A GPU or extra recovery')
|
||||
for key in ('jobs', 'rows', 'training_root', 'training_contract', 'comparison_contract_sha256',
|
||||
'postfit_gpu_cap', 'postfit_cpu_cap', 'item_gpu_cap', 'new_fits', 'new_decodes'):
|
||||
if c[key] != old[key]:
|
||||
raise ValueError('Original science/budget differs: ' + key)
|
||||
if c['postfit_root'] == old['postfit_root']:
|
||||
raise ValueError('Original stopped root must remain untouched')
|
||||
for p, h in old['code_sha256'].items():
|
||||
if c['code_sha256'].get(p) != h:
|
||||
raise ValueError('Original pinned source changed')
|
||||
|
||||
|
||||
def validate_stop(done):
|
||||
records = done['records']
|
||||
if done['status'] != 'STOPPED' or len(records) != 2 or [x['index'] for x in records] != [0, 1]:
|
||||
raise ValueError('Unexpected original stop scope')
|
||||
first, failed = [x['guard'] for x in records]
|
||||
if first['status'] != 'PASS' or first['monitor_errors'] or not first['worker'] or first['worker']['status'] != 'PASS' or first['worker'].get('monitor_errors') != 0:
|
||||
raise ValueError('Predict0 was not a clean PASS')
|
||||
if failed['status'] != 'STOPPED' or failed['worker'] is not None or 'RuntimeError: SELECTED_GPU_BUSY_NO_RETRY' not in failed['reasons']:
|
||||
raise ValueError('Not the authorized zero-worker infrastructure stop')
|
||||
for key in ('prediction_gpu_seconds', 'postfit_cpu_seconds', 'training_gpu_seconds_once'):
|
||||
if not math.isfinite(done[key]) or done[key] < 0:
|
||||
raise ValueError('Invalid original ledger')
|
||||
|
||||
|
||||
def six_receipts(c):
|
||||
source = Path(c['original_root']) / 'tron-six-training-receipts-check.json'
|
||||
check = read(source)
|
||||
if check['status'] != 'PASS' or [r['job'] for r in check['rows']] != list(range(6)):
|
||||
raise ValueError('Original six-receipt check missing')
|
||||
for row in check['rows']:
|
||||
p = Path(c['training_root']) / 'jobs' / str(row['job']) / 'guards/train.receipt.json'
|
||||
if str(p) != row['path']:
|
||||
raise ValueError('Original training receipt path mismatch')
|
||||
receipt = read(checked(p, row['sha256']))
|
||||
if type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0:
|
||||
raise ValueError('Training monitoring receipt not clean')
|
||||
guard = read(p.parent.parent / 'complete.json')
|
||||
if guard['worker'] != receipt:
|
||||
raise ValueError('Embedded training receipt differs')
|
||||
return check
|
||||
|
||||
|
||||
def config(root):
|
||||
c, tc, pilot = original_config(root)
|
||||
original = Path(c['original_root'])
|
||||
old, _, _ = original_config(original)
|
||||
validate_scope(c, old)
|
||||
done = read(checked(original / 'complete.json', c['original_complete_sha256']))
|
||||
validate_stop(done)
|
||||
if any((original / n).exists() for n in ('score', 'predictions-manifest.json')):
|
||||
raise ValueError('Original scoring or full prediction barrier already exists')
|
||||
for k, name in [('prior_postfit_gpu', 'prediction_gpu_seconds'), ('prior_postfit_cpu', 'postfit_cpu_seconds'),
|
||||
('training_gpu_once', 'training_gpu_seconds_once')]:
|
||||
if c[k] != done[name]:
|
||||
raise ValueError('Original cost must be retained exactly once')
|
||||
six_receipts(c)
|
||||
return c, tc, pilot
|
||||
|
||||
|
||||
def adoption(c, tc):
|
||||
current = terminal_inputs(c, tc)
|
||||
original = Path(c['original_root'])
|
||||
verify(original, 'models')
|
||||
old = read(original / 'models-adopted.json')
|
||||
if old != current or current['training_gpu_seconds_once'] != c['training_gpu_once']:
|
||||
raise ValueError('Predict0 and resumed jobs must use identical frozen models')
|
||||
return current
|
||||
|
||||
|
||||
def reuse_entries(c, tc, *, verify_bytes=True):
|
||||
original = Path(c['original_root'])
|
||||
manifest = read(checked(original / 'jobs/0/prediction-records/manifest.json', c['predict0_manifest_sha256']))
|
||||
result = read(original / 'jobs/0/prediction-records/result.json')
|
||||
if result['status'] != 'COMPLETE' or result['targets_read'] or result['parts'] != c['jobs'][0]['expected_parts']:
|
||||
raise ValueError('Predict0 result mismatch')
|
||||
entries = manifest['entries']
|
||||
f = tc['pools']['25']
|
||||
domains = {'learning': 'own_learning', 'validation': 'common_validation', 'test': 'common_test'}
|
||||
expected = {(arm, case): domain for role, domain in domains.items() for case in f[role]
|
||||
for arm in ('epoch25', 'c25F')}
|
||||
if len(entries) != len(expected) or {(e['arm'], e['case']) for e in entries} != set(expected):
|
||||
raise ValueError('Complete predict0 model and clock coverage required')
|
||||
if manifest['targets_read'] or result['entries'] != len(entries):
|
||||
raise ValueError('Predict0 targets/entry count')
|
||||
for e in entries:
|
||||
p = original / 'predictions' / e['arm'] / (e['case'] + '.npz')
|
||||
if (e['path'] != str(p) or e['pool'] != 25 or e['domain'] != expected[(e['arm'], e['case'])]
|
||||
or e['component_id'] != c['rows'][e['case']]['component_id']
|
||||
or e['origin'] != 'ITEM1_FROZEN_ENDPOINT_OR_CLOCK'):
|
||||
raise ValueError('Predict0 original identity')
|
||||
if verify_bytes:
|
||||
checked(p, e['sha256'])
|
||||
return entries
|
||||
|
||||
|
||||
def cumulative_cost(c, new_gpu, new_cpu):
|
||||
if any(not math.isfinite(v) or v < 0 for v in [new_gpu, new_cpu]):
|
||||
raise ValueError('Invalid new cost ledger')
|
||||
gpu = c['prior_postfit_gpu'] + new_gpu
|
||||
return dict(prediction_gpu_seconds=gpu, postfit_cpu_seconds=c['prior_postfit_cpu'] + new_cpu,
|
||||
training_gpu_seconds_once=c['training_gpu_once'],
|
||||
cumulative_item_gpu_seconds=c['training_gpu_once'] + gpu)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Recovery-only guard: bounded pre-worker idle wait inside original900s cap."""
|
||||
import json,os,signal,sys,threading,time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
||||
out=list(reasons)
|
||||
def add(s):
|
||||
if s not in out:out.append(s)
|
||||
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
||||
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
||||
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
||||
return out
|
||||
|
||||
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
||||
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase3_item1_postfit_worker.py','predict',str(queue),str(fold)]
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def scan():
|
||||
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
||||
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
||||
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
||||
try:
|
||||
wait_start=time.monotonic();waiting=[]
|
||||
while True:
|
||||
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
||||
if requested:raise RuntimeError('Signal received during idle wait')
|
||||
duration=time.monotonic()-wait_start
|
||||
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
if mem<=128*2**20 and util==0:break
|
||||
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
||||
for i in range(3):
|
||||
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'prediction-idle-polls.json',samples)
|
||||
if requested:raise RuntimeError('Signal received during setup')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=900-elapsed()-10
|
||||
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
||||
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
result=None;p=root/'prediction-records/result.json'
|
||||
try:
|
||||
if p.exists():result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
||||
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_item1_gpu_seconds=prior_gpu+duration,no_retry=True)
|
||||
save(root/'prediction-guard.json',out);return out
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Metadata and opaque-hash preparation; no new prediction or target access."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_item1_postfit_common import config as old_config, read
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, six_receipts, adoption, reuse_entries
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
B = Path(__file__).resolve().parents[1]
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
R = O.parent / 'phase3-item1-postfit-recovery-2026-11-18'
|
||||
|
||||
|
||||
def prepare():
|
||||
start = time.process_time()
|
||||
if R.exists():
|
||||
raise FileExistsError('Existing recovery entry: inspect before any further action')
|
||||
old, tc, pilot = old_config(O)
|
||||
validate_review(O)
|
||||
done = read(O / 'complete.json')
|
||||
validate_stop(done)
|
||||
sources = dict(old['source_sha256'])
|
||||
for name in ['contract.json', 'complete.json', 'started.json', 'resource-progress.json',
|
||||
'tron-six-training-receipts-check.json', 'models-adopted.json', 'models-git-freeze.json',
|
||||
'review.json', 'implementation-git-freeze.json',
|
||||
'jobs/0/prediction-guard.json', 'jobs/0/prediction-records/manifest.json',
|
||||
'jobs/0/prediction-records/result.json', 'jobs/0/prediction-records/environment.json']:
|
||||
sources[str(O / name)] = sha(O / name)
|
||||
codes = dict(old['code_sha256'])
|
||||
additional = local_code_closure([B / 'tools' / n for n in ['phase3_item1_resume.py',
|
||||
'phase3_item1_resume_common.py', 'phase3_item1_resume_guard.py', 'prepare_phase3_item1_resume.py',
|
||||
'test_phase3_item1_resume.py']])
|
||||
for p, h in additional.items():
|
||||
if p in codes and codes[p] != h:
|
||||
raise ValueError('Old pinned code changed')
|
||||
codes[p] = h
|
||||
c = dict(old, postfit_root=str(R), code_sha256=codes, source_sha256=sources,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
original_root=str(O), original_complete_sha256=sha(O / 'complete.json'),
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1, 7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
predict0_manifest_sha256=sha(O / 'jobs/0/prediction-records/manifest.json'),
|
||||
prior_postfit_gpu=done['prediction_gpu_seconds'], prior_postfit_cpu=done['postfit_cpu_seconds'],
|
||||
training_gpu_once=done['training_gpu_seconds_once'],
|
||||
engineering_revision='Single labelled predict1..6 continuation; immutable predict0 by hash',
|
||||
same_independent_review=False)
|
||||
validate_scope(c, old)
|
||||
six_receipts(c)
|
||||
adopted = adoption(c, tc)
|
||||
entries = reuse_entries(c, tc)
|
||||
env = read(O / 'jobs/0/prediction-records/environment.json')
|
||||
if env['checkpoint_sha256'] != adopted['models'][0]['checkpoint_sha256']:
|
||||
raise ValueError('Predict0 checkpoint differs')
|
||||
R.mkdir()
|
||||
save(R / 'contract.json', c)
|
||||
save(R / 'preparation-receipt.json', dict(status='PREPARED_PENDING_NARROW_DELTA_REVIEW',
|
||||
metadata_and_hash_cpu_seconds=time.process_time() - start, reuse_files=len(entries),
|
||||
reuse_bytes=sum(Path(e['path']).stat().st_size for e in entries), original_models_exact_match=True,
|
||||
six_monitor_errors_all_integer_zero=True, old_sources_unchanged=True,
|
||||
original_complete_sha256=c['original_complete_sha256'], contract_sha256=sha(R / 'contract.json'),
|
||||
no_model_deserialization=True, no_prediction_array_decode=True, target_values_read=False,
|
||||
new_prediction=False, new_score=False, launched=False))
|
||||
print(read(R / 'preparation-receipt.json'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prepare()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
|
||||
import copy
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase3_item1_postfit_common import read, require_budget
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
|
||||
from phase3_item1_resume import install_reuse
|
||||
import phase3_item1_resume_guard as guard
|
||||
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
|
||||
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = read(O / 'contract.json')
|
||||
self.done = read(O / 'complete.json')
|
||||
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
|
||||
training_gpu_once=self.done['training_gpu_seconds_once'])
|
||||
|
||||
def test_scope_and_actual_original_stop(self):
|
||||
validate_scope(self.c, self.old)
|
||||
validate_stop(self.done)
|
||||
|
||||
def test_no_predict0_rerun(self):
|
||||
self.c['resume_indices'] = list(range(7))
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_gpu1_only(self):
|
||||
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
|
||||
self.c['gpu_uuid'] = uuid
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_no_budget_change(self):
|
||||
self.c['postfit_gpu_cap'] = 7201
|
||||
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_failed_job_worker_must_be_null(self):
|
||||
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_predict0_must_be_pass(self):
|
||||
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_old_cost_once_not_per_job(self):
|
||||
a = cumulative_cost(self.c, 100., 200.)
|
||||
b = cumulative_cost(self.c, 300., 500.)
|
||||
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
|
||||
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
|
||||
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
|
||||
|
||||
def test_old_cost_limits_remaining_reserve(self):
|
||||
d = cumulative_cost(self.c,60,240)
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
|
||||
with self.assertRaises(RuntimeError):
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
|
||||
|
||||
def test_original_six_receipts_currently_clean(self):
|
||||
self.assertEqual(len(six_receipts(self.c)['rows']),6)
|
||||
|
||||
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
|
||||
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
|
||||
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
|
||||
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
|
||||
with self.assertRaises(FileExistsError):install_reuse(root,[e])
|
||||
|
||||
def test_reuse_bad_hash_rejected_before_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
|
||||
with self.assertRaises(ValueError):
|
||||
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
|
||||
self.assertEqual(list(root.iterdir()),[])
|
||||
|
||||
def guarded(self, mode):
|
||||
timer=[0.];calls=[0]
|
||||
def sleep(seconds): timer[0]+=seconds
|
||||
def probe(uuid):
|
||||
calls[0]+=1
|
||||
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
|
||||
return 0,0
|
||||
class Thread:
|
||||
def __init__(self,*a,**k):pass
|
||||
def start(self):pass
|
||||
def join(self,**k):pass
|
||||
def is_alive(self):return False
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
|
||||
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
|
||||
def supervisor(*a,**k):
|
||||
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
|
||||
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
|
||||
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
|
||||
result=guard.run_prediction(root,c,{},1,0,300)
|
||||
return result,sup.call_count,read(out/'recovery-idle-wait.json')
|
||||
|
||||
def test_busy_then_idle_counts_wait_and_three_polls(self):
|
||||
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
|
||||
self.assertEqual(r['gpu_seconds_charged'],65.)
|
||||
self.assertEqual(len(w['samples']),3)
|
||||
|
||||
def test_busy_timeout_stops_without_worker(self):
|
||||
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
|
||||
self.assertEqual(r['gpu_seconds_charged'],80.)
|
||||
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
|
||||
|
||||
def test_supervisor_monitor_error_still_rejected(self):
|
||||
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
|
||||
|
||||
def test_wait_limit_cannot_be_relaxed(self):
|
||||
self.c['idle_wait_seconds']=21
|
||||
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
Reference in new issue
Block a user