From 2dc2f39e376c456d41469e0b4de32b5120d5e8ca Mon Sep 17 00:00:00 2001 From: MeltAstra Date: Wed, 18 Nov 2026 16:50:51 +0800 Subject: [PATCH] Repair seed1 v2 observation coverage and preserve original BLOCKED review --- .../evidence.json | 146 + .../reproduce_observation_gap.py | 60 + .../review.json | 98 + .../expected-review-binding.json | 59 + .../implementation-authorization.json | 4 + .../implementation-git-freeze.json | 121 + .../attempt-01.log | 17 + ...se2-h2-s2-execution-contract-2026-11-17.md | 17 + .../docs/phase3-seed1-v2-entry-2026-11-18.md | 23 + .../comparison-contract.json | 2866 ++++++++++++++ .../phase3-seed1-v2-2026-11-18/contract.json | 3449 +++++++++++++++++ .../preparation-audit.json | 33 + .../tools/freeze_g1_components.py | 314 ++ .../tools/gre24_models.py | 42 + .../tools/phase2_fixed_io.py | 34 + .../tools/phase2_gate_outcome_review.py | 20 + .../tools/phase2_h1_worker_v2.py | 179 + .../tools/phase2_h2_data.py | 87 + .../tools/phase2_h2_model.py | 108 + .../tools/phase2_h2_pilot_worker.py | 80 + .../tools/phase2_h2_prepare.py | 63 + .../tools/phase2_h2_s1_common.py | 30 + .../tools/phase2_h2_s1_guard.py | 83 + .../tools/phase2_h2_s1_launch.py | 20 + .../tools/phase2_h2_s1_worker.py | 62 + .../tools/phase2_h2_s2_common.py | 55 + .../tools/phase2_h2_s2_guard.py | 86 + .../tools/phase2_h2_s2_prepare.py | 51 + .../tools/phase2_h2_s2_queue.py | 100 + .../tools/phase2_h2_s2_worker.py | 67 + .../tools/phase2_home_freeze_io.py | 29 + .../tools/phase2_local_freeze_io.py | 35 + .../tools/phase2_residual_math.py | 65 + .../tools/phase3_seed1_common.py | 116 + .../tools/phase3_seed1_continuation_common.py | 127 + .../tools/phase3_seed1_continuation_guard.py | 88 + .../tools/phase3_seed1_continuation_queue.py | 212 + .../tools/phase3_seed1_continuation_worker.py | 67 + .../tools/phase3_seed1_guard.py | 88 + .../tools/phase3_seed1_queue.py | 212 + .../tools/phase3_seed1_v2_common.py | 147 + .../tools/phase3_seed1_v2_guard.py | 88 + .../tools/phase3_seed1_v2_observation.py | 34 + .../tools/phase3_seed1_v2_queue.py | 214 + .../tools/phase3_seed1_v2_worker.py | 67 + .../tools/phase3_seed1_worker.py | 67 + .../tools/prepare_phase3_seed1.py | 43 + .../prepare_phase3_seed1_continuation.py | 43 + .../tools/prepare_phase3_seed1_v2.py | 47 + .../tools/s2_gpu_idle_gate.py | 53 + .../tools/s2_main_memory_guard.py | 133 + .../tools/s2_models/network.py | 119 + .../tools/s2_pilot_train.py | 79 + .../tools/s2_precision_continuation_guard.py | 122 + .../tools/s2_recovery_gpu_supervisor.py | 158 + .../tools/s2_recovery_monitoring.py | 40 + .../tools/test_phase2_h2_s2.py | 97 + .../tools/test_phase3_seed1.py | 105 + .../tools/test_phase3_seed1_continuation.py | 62 + .../tools/test_phase3_seed1_v2.py | 91 + .../tools/track_a_h2_common.py | 144 + .../tools/track_a_h2_prepare.py | 108 + .../original-review/evidence.json | 146 + .../reproduce_observation_gap.py | 60 + .../original-review/review.json | 98 + .../preservation.json | 291 ++ .../tests-attempt-01.log | 22 + .../tests-attempt-02.log | 23 + docs/collaboration-stage-review-policy.md | 3 + docs/phase2-live-handoff-2026-11-17.md | 4 + ...se3-seed1-v2-observation-fix-2026-11-18.md | 13 + .../phase3-seed1-v2-2026-11-18/contract.json | 9 +- .../expected-review-binding.json | 12 +- .../implementation-authorization.json | 4 +- .../implementation-git-freeze.json | 146 +- .../observation-fix-audit.json | 21 + tools/phase3_seed1_v2_observation.py | 46 +- tools/phase3_seed1_v2_queue.py | 20 +- tools/test_phase3_seed1_v2.py | 84 + 79 files changed, 12451 insertions(+), 25 deletions(-) create mode 100644 artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json create mode 100644 artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py create mode 100644 artifacts/meltassist-seed1-v2-review-2026-11-18/review.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log create mode 100644 artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log create mode 100644 docs/phase3-seed1-v2-observation-fix-2026-11-18.md create mode 100644 home-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json diff --git a/artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json b/artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json new file mode 100644 index 000000000..9a8573c3a --- /dev/null +++ b/artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json @@ -0,0 +1,146 @@ +{ + "timestamp": "2026-11-18T08:43:50.686415+00:00", + "binding_sha256": "b5baee370b6cabf45ef495c6d4ca37949486b3ac346d6ae304d215ccd904fc9c", + "hash_audit": { + "code_files": 51, + "source_files": 407, + "local_implementation_files": 56, + "old_stop_files": [ + 108, + 192 + ], + "all_current_and_local_git_proofs": "PASS", + "old_guards_and_assignments_match": true + }, + "independent_tests": { + "count": 12, + "status": "PASS", + "failures": 0, + "errors": 0, + "wall_seconds": 0.022 + }, + "additional_projection_boundaries": [ + "measured median36000 dominates initial reserve for unmeasured active fold", + "unmeasured active actual35000 dominates initial reserve and measured20000; resulting started median27500" + ], + "config": "PASS; metadata/hash/version checks only", + "initial_projection": { + "status": "PASS", + "rows": [ + { + "id": 0, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 1, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 2, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 3, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 4, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + } + ], + "projected_gpu_seconds": 158717.68659210298, + "phase3_prior_gpu_once": 80903.62940728385, + "projected_phase3_gpu_seconds": 239621.31599938683, + "postfit_reserve": 7200, + "pending_reserve_gpu_seconds": 30303.537318420596, + "actual_runaway_gpu_seconds": 40000, + "total_projection_within_cap": true + }, + "ledger": { + "old_attempts": [ + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18", + "gpu_seconds": 539.0720570000121 + }, + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18", + "gpu_seconds": 7943.241350897006 + } + ], + "old_seed1_once": 8482.313407897018, + "new_training_cap": 151517.68659210298, + "new_total_cap": 158717.68659210298, + "postfit": 7200, + "phase3_prior": 80903.62940728385, + "cumulative_seed1_cap": 167200 + }, + "unchanged_science": "Worker and guard byte-identical after renaming imports/child entry; all job fields except CPU/GPU caps match original; six role/idle/release helpers unchanged", + "observation_gap": { + "reproducer": "/home/frank14f/optane/MeltMind/artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py", + "sha256": "ace732bf9beaece291a56b342739a6c8e9e528b27c889132baacf0a8a38819cb", + "source_sha256": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "method": "Execute exact collect and active epoch-loop AST fragments against temporary synthetic histories, mocked process and observation callback; no GPU query or production files", + "cases": [ + { + "scenario": "exited_before_next_poll", + "last_observed": 199, + "completed": [ + 199, + 200 + ], + "expected": [ + 200 + ], + "actual": [], + "missing": [ + 200 + ] + }, + { + "scenario": "two_unseen_epoch_receipts", + "last_observed": 198, + "completed": [ + 199, + 200 + ], + "expected": [ + 199, + 200 + ], + "actual": [ + 200 + ], + "missing": [ + 199 + ] + } + ] + }, + "root_markers": { + "review.json": false, + "started.json": false, + "complete.json": false + }, + "publication": "Small package publication not independently verified in this review", + "science_executed": false, + "real_gpu_query": false, + "target_values_loaded": false, + "model_values_loaded": false, + "source_edits": false, + "git_writes": false +} diff --git a/artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py b/artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py new file mode 100644 index 000000000..bfb9aa877 --- /dev/null +++ b/artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py @@ -0,0 +1,60 @@ +"""Replay exact coordinator AST fragments using synthetic files/processes only.""" +import ast +import io +import json +from pathlib import Path +import tempfile +import time +from types import SimpleNamespace +from phase3_seed1_v2_common import project_cell, project_total +from phase2_h1_worker_v2 import sha, save + +source = Path('tools/phase3_seed1_v2_queue.py') +tree = ast.parse(source.read_text()) +run = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'run') +collect = next(n for n in run.body if isinstance(n, ast.FunctionDef) and n.name == 'collect') +epoch_loop = next(n for n in ast.walk(run) if isinstance(n, ast.For) + and isinstance(n.iter, ast.Name) and n.iter.id == 'active') + +def compiled(node): + return compile(ast.fix_missing_locations(ast.Module(body=[node], type_ignores=[])), str(source), 'exec') + +results = [] +for scenario in ['exited_before_next_poll', 'two_unseen_epoch_receipts']: + with tempfile.TemporaryDirectory() as td: + root = Path(td) + work = root / 'jobs/0' + history = work / 'worker/history' + history.mkdir(parents=True) + save(root / 'contract.json', {}) + save(work / 'assignment.json', {}) + save(work / 'complete.json', dict(status='PASS', monitor_errors=[], + worker=dict(monitor_errors=0), gpu_seconds_charged=1000)) + for epoch in [199, 200]: + save(history / f'{epoch:03d}.json', dict(epoch=epoch, train_epoch_seconds=1)) + v = dict(process=SimpleNamespace(poll=lambda: 0), start=time.monotonic(), + log=io.StringIO(), job=dict(gpu_uuid='SYNTHETIC-GPU')) + calls = [] + jobs = [dict(id=i, setup_gpu_seconds=60, gpu_seconds=40000, + validation_seconds_bound=1) for i in range(5)] + env = dict(root=root, active={0:v}, terminal={}, last_epoch={0:199 if scenario.startswith('exited') else 198}, + c=dict(jobs=jobs, train_gpu_cap=151517.68659210298, + postfit_gpu_reserve=7200, gpu_cap=158717.68659210298, + phase3_prior_gpu_once=80903.62940728385, phase3_gpu_cap=612000, + runaway_gpu_seconds=40000), estimates={}, + time=time, read=lambda p:json.loads(p.read_text()), save=save, sha=sha, + stop=lambda reason:None, active_charges=lambda:{0:1000}, + epoch_host_observation=lambda root,i,e,gpu:calls.append(e), + project_cell=project_cell, project_total=project_total) + exec(compiled(collect), env) + if scenario.startswith('exited'): + assert env['collect'](0,v) is True + exec(compiled(epoch_loop), env) + expected = [200] if scenario.startswith('exited') else [199,200] + assert calls != expected + results.append(dict(scenario=scenario,expected_epoch_observations=expected, + actual_epoch_observations=calls, + terminal_ids=list(env['terminal']), + missing=sorted(set(expected)-set(calls)))) +print(json.dumps(dict(source_sha256=sha(source),reproductions=results, + scientific_workload=False,real_gpu_query=False),indent=2)) diff --git a/artifacts/meltassist-seed1-v2-review-2026-11-18/review.json b/artifacts/meltassist-seed1-v2-review-2026-11-18/review.json new file mode 100644 index 000000000..6073a7c58 --- /dev/null +++ b/artifacts/meltassist-seed1-v2-review-2026-11-18/review.json @@ -0,0 +1,98 @@ +{ + "reviewed_commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "implementation_proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8", + "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + }, + "verdict": "BLOCKED", + "reviewer": "MeltAssist", + "timestamp": "2026-11-18T08:43:50.686415+00:00", + "review_scope": "Seed1 v2 stopping-rule, ledger and required epoch observation review; directly read Claude1794990750/0899/0914", + "evidence_path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json", + "evidence_sha256": "219bc5bb74575ae880803a89bf58bb1ae65dd55dc618c3710680d9d8af99371c", + "findings": [ + { + "id": "V2-EPOCH-OBSERVATION-COVERAGE", + "priority": "P2", + "file": "tools/phase3_seed1_v2_queue.py", + "line": 115, + "related_lines": [ + 79, + 126, + 131, + 134 + ], + "title": "Drain completed-epoch observation receipts before removing a terminal fold; do not silently skip a backlog", + "requirement": "Approved v2 records GPU process/host observations for every completed epoch, retaining incomplete observation evidence without causal claims.", + "observed": "The loop collects exited guards first and deletes their active entry. The observation loop only visits remaining active entries and only rows[-1]. Thus epoch200 can disappear when the guard exits before the next poll; multiple unseen completed epochs collapse to the latest. Exact coordinator-fragment CPU replay reproduced both omissions. No actual scientific run was used.", + "requested_fix": "Track observation coverage separately from cost-projection progress. Before terminal removal/global-stop cleanup, account for every completed-epoch receipt; handle all previously unseen epochs, not just latest. Record true snapshot time and explicitly mark delayed/unobserved history OBSERVATION_INCOMPLETE or equivalent rather than invent past GPU occupancy. Preserve projection formula, actual charging, idle dispatch and foreign-process nonintervention. Add terminal-before-poll and multi-receipt-backlog synthetic coverage." + } + ], + "passed_checks": [ + "Old539.0720570000121+7943.241350897006=8482.313407897018 counted once inside167200; new total158717.68659210298/train151517.68659210298+7200; phase3prior80903.62940728385 already includes both old attempts", + "Total projection uses started projections/terminal actual plus pending started-median reservations; initial train/5; active unmeasured max(actual,initial,measured median), then contributes to started median; no double addition of terminal actual", + "Measured max-epoch1.25/validation/setup formula unchanged; projection>40000 does not independently STOP; actual40000 guard remains with conservative setup/termination margins", + "Observation subprocess uses5s timeout, preserves query/identity errors as OBSERVATION_INCOMPLETE; return value not used by dispatch or projection and elapsed collection time is charged", + "Three-generation duplicate detection, shared GPU locks, actual-idle dispatch/GPU2 priority, scientific recipe and CPU/RSS/VRAM/disk protections remain", + "Independent12tests and two additional unmeasured-reserve boundary checks PASS; config/initial projection PASS;51code/407source/56implementation and both old108/192freeze-file proofs verified" + ], + "limitations": [ + "This is an observation coverage blocker, not a ledger failure, scientific failure or evidence of host contention", + "Prior epoch34 slowdown cause remains unproven; current or delayed snapshots cannot establish historical causality", + "Original stopped coordinator CPU accounting gaps preserved, not estimated", + "No launch or result claimed; old PASS cannot substitute for a new exact v2 verdict; postfit remains separately reviewed", + "No remote publication raw-byte verification claimed; local immutable identity verified" + ] +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json new file mode 100644 index 000000000..86b1edfdd --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json @@ -0,0 +1,59 @@ +{ + "reviewed_commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "implementation_proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8", + "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + } +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json new file mode 100644 index 000000000..42e220e40 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json @@ -0,0 +1,4 @@ +{ + "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8" +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json new file mode 100644 index 000000000..7cb699cde --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json @@ -0,0 +1,121 @@ +{ + "commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "files": { + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "af9531defc4289d021eec3809d87ba4f7497927f94be78795027ca9248f21f8f", + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073", + "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-entry-2026-11-18.md": "2ed35f7b8cb34fab947db736c6f13c4aaecf48513fbeb65d7b97aa9a31e93158", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "65083e1e5ffb5b2514a2eabc0b83fd3ff4430e01530ff0b15c0478a26bf826bb" + }, + "git_paths": { + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "home-runs/phase3-seed1-v2-2026-11-18/contract.json", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json", + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "tools/phase3_seed1_common.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "tools/phase3_seed1_continuation_common.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "tools/phase3_seed1_v2_observation.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "tools/phase3_seed1_v2_queue.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "tools/prepare_phase3_seed1_v2.py", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "tools/test_phase3_seed1_v2.py", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "tools/phase3_seed1_continuation_guard.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "tools/phase3_seed1_continuation_queue.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "tools/phase3_seed1_continuation_worker.py", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "tools/prepare_phase3_seed1_continuation.py", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "tools/test_phase3_seed1_continuation.py", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "tools/phase3_seed1_guard.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "tools/phase3_seed1_queue.py", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "tools/phase3_seed1_worker.py", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "tools/prepare_phase3_seed1.py", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "tools/test_phase3_seed1.py", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md", + "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-entry-2026-11-18.md": "docs/phase3-seed1-v2-entry-2026-11-18.md", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log" + }, + "remote_published": false, + "local_repository": "/tmp/meltmind-phase2-local-fixed-20261117" +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log new file mode 100644 index 000000000..36acd458a --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log @@ -0,0 +1,17 @@ +test_active_unmeasured_not_free_and_actual_not_ignored (test_phase3_seed1_v2.V2Tests) ... ok +test_actual_runaway_stops_within_total (test_phase3_seed1_v2.V2Tests) ... ok +test_both_old_attempts_once_and_initial_budget (test_phase3_seed1_v2.V2Tests) ... ok +test_epoch_snapshot_marks_foreign_without_intervening (test_phase3_seed1_v2.V2Tests) ... ok +test_even_median_and_two_started_count_once (test_phase3_seed1_v2.V2Tests) ... ok +test_one_measured_fold_sets_pending_median (test_phase3_seed1_v2.V2Tests) ... ok +test_projected_above40000_does_not_alone_stop (test_phase3_seed1_v2.V2Tests) ... ok +test_projection_math_unchanged (test_phase3_seed1_v2.V2Tests) ... ok +test_scientific_entry_and_guard_only_names_changed (test_phase3_seed1_v2.V2Tests) ... ok +test_snapshot_error_retained_observation_only (test_phase3_seed1_v2.V2Tests) ... ok +test_terminal_actual_replaces_prior_estimate (test_phase3_seed1_v2.V2Tests) ... ok +test_total_exceed_stops_even_below_runaway (test_phase3_seed1_v2.V2Tests) ... ok + +---------------------------------------------------------------------- +Ran 12 tests in 0.021s + +OK diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md new file mode 100644 index 000000000..480d04b12 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md @@ -0,0 +1,17 @@ +# H2-S2: four new fixed folds, immutable fold0 reuse + +Claude1794923791.467409 authorizes folds1–4 in parallel, same H2 recipe/seed0/200epochs, within the remaining original144000GPU-second budget. Claude1794923933.456449 accepts the stage corrections: S2 is approved but does not become running until actual worker evidence exists; S2 precedes data-sufficiency. This entry implements only the four training jobs and checkpoint freeze. It starts no prediction, pooled scorer, data-sufficiency run, refit of fold0, or retry. + +Original S1 settled usage is18858.21490871499GPU-s; remaining125141.78509128501. Reserve2400GPU-s for four later prediction jobs at600each (including setup/idle/failure). Training guard/coordinator allocation is122741.78509128501GPU-s, divided into four30685.446272821252GPU-s job ceilings. Every started fold charges60setup plus all elapsed time from dispatch until guard exit, including normalization, import, first-epoch barrier, validation, checkpoints, idle checks and failures. Settled charge is the maximum of coordinator observed elapsed and the guard's actual receipt. Prior pilot/train/S1 postfit is added once; completed fold0 is never charged or executed anew. Guards retain the same one-process/16GiB treeRSS/15GiB recorded-or-deviceVRAM/20GiB output boundaries and an engineering CPU cap equal to their GPU ceiling. Four jobs jointly have64GiB RSS and80GiB output ceilings. Sampling overshoot is recorded and fails, not clipped. + +GPU UUIDs are fixed to the four V100s read on Horsea. Worker CPUs0–1/2–3/4–5/6–7, fold guards8/9/10/11, coordinator12–13 avoid the original gate queue's16–27 workers and30–31 coordinator. Each fold requires three idle polls on its own GPU; busy stops without retry. Global stop propagates to all active guards and they kill/reap their own workers. The coordinator has no detached training continuation and checks actual source identities again at terminal. + +Each new fold explicitly selects its original immutable fold record; the original preprocessing helper reads that selected record from its local slot0 while its embedded fold identity remains1–4. Roles are never renamed. Learning-only19-feature normalization is recomputed independently for the original learning set of that fold using the unchanged FP64 Welford population statistics. Validation is diagnostic every10epochs, exactly as S1, with no checkpoint selection. The original FP32 model/aggregations, FP64 F skip, original fold learning clock, seed0, shuffling, optimizer, clipping, learning-rate schedule, full-part loss and epoch loop remain unchanged. Only last200 is saved. Test cases cannot enter that fold's learning or validation loader. + +After each fold's first complete epoch, the worker waits for a shared exact-contract PASS. The coordinator requires all four actual epoch receipts, with original part/node counts and optimizer steps. It projects each fold as charged elapsed +1.25×(remaining epochs×largest measured training-epoch time + remaining validation events×validation bound)+60tail. The initial validation bound is S1's maximum actual diagnostic time12.383006267948076s scaled by the maximum of that fold's validation node/edge/part ratios. Later actual validation times can only raise that bound. All four projected jobs must fit their fixed ceilings and old+allnew+2400prediction must fit144000. Checks refresh after each common ten-epoch milestone. A failed cost gate, job, monitor, signal, source, resource or terminal-identity check stops the queue and preserves attempts; no retry or automatic change in limits/recipe. + +On four PASS terminals, last200 checkpoints, per-fold normalization and a model manifest are frozen. The queue stops at TRAIN_COMPLETE_POSTFIT_PENDING. Future prediction/scoring receives a separate entry and exact MeltAssist pre-score semantic PASS; the S1 receipt does not authorize a different scorer. New postfit CPU cap3600 and prediction reserve2400 remain recorded, not spent by this entry. No test target arrays are read by preparation, acceptance or this training queue; only original learning and validation targets for each active fold are allowed. + +Confirmation is fixed before any S2 result: all552held-out parts/204components; per-arm whole-component means and equal component weights, paired intersection,2000paired component bootstrap/seed2026111408, pooled five-fold field H2−A CI upper<0. Undefined/missing complete field support cannot confirm. B13-E1b comparisons and last-location/tau are reported but do not gate. Fold0 predictions and metrics are reused by hash; no fold0 prediction or score rerun. This confirmation follows observation of fold0 and is not an untouched five-fold preregistration. It cannot by itself isolate connectivity causality. Original negative predictions, NA and limits remain visible. + +Acceptance is16CPU synthetic tests: exact scientific epoch-loop AST except the first-epoch wait, fold roles/learning-only normalization, blocked test I/O, original-cost-once/four-wait/prediction reservation arithmetic, per-job/global projection failures, exact release binding, strict complete confirmation rule, process identities and retained failure peaks. Two lifecycle tests use a synthetic supervisor/GPU probe and temporary outputs to exercise the actual guard control flow without CUDA. These stand-ins test engineering transitions, not real numerical training. Attempt1's missing synthetic parts fixture and prelaunch discovery of the fold guard contract path were repaired before any real launch; the failed log and final16PASS log are retained. No scientific code from S1 was modified. diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md new file mode 100644 index 000000000..10f5909f5 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md @@ -0,0 +1,23 @@ +# Seed1 v2: total projection and fixed cumulative budget + +The preceding original and labelled-continuation seed1 runs are terminal; neither is resumed or rewritten. Claude1794990750.013949 authorizes an independent v2 binding, clarified by1794990899.609759 and1794990914.918719. This entry changes the stopping rule and adds epoch-boundary host observations, preserving the same five original grouped folds, seed1, fresh initialization,200 epochs, learning-only normalization, optimizer and learning-rate schedule, last-epoch-only checkpoint, diagnostic validation, dynamic idle dispatch and GPU2 Track A postfit priority. Folds0/1 restart fresh,2/3/4 are first attempts. No partial checkpoint adoption, seed2, ensemble or further seed1 continuation is allowed. + +## Ledger and stopping logic + +Old original539.0720570000121 + old continuation7943.241350897006 =8482.313407897018 GPU-s counted exactly once inside the unchanged cumulative seed1 cap167200. New v2 cap158717.68659210298 consists of training151517.68659210298 +postfit7200. phase3_prior80903.62940728385 already contains original item1 and both old seed1 attempts; add only new v2 cost to it. Phase3 ceiling612000 remains. Runtime counters are new-v2 only. The original stopped-coordinator CPU accounting gaps remain explicit. + +For a measured active fold, projected cost remains spent +1.25×((200−completed_epoch)×maximum observed epoch time +remaining validation-count×maximum of original/observed validation timing)+120. This preserves the original timing formula. Completed-fold actual cost replaces its former projection. A projection is never allowed below its currently charged actual cost. + +Never-started folds reserve the median of current started-fold projections. Before any epoch measurement, initial reserve is151517.68659210298/5 =30303.537318420596 per fold. For a started fold still awaiting its first measured epoch, use the conservative maximum of its actual charge, initial equal share, and measured-fold median, then include that provisional value in the started-fold median. This explicit intermediate-state rule avoids treating an active but unmeasured fold as zero work; it is included in the narrow review. A completed started fold contributes its actual cost to that median. + +Total projection is the sum of started-fold projected/actual costs, pending-fold reservations, and7200 postfit. It must fit the new total158717.68659210298 and phase3 total612000. Per-fold projections are recorded but do not independently stop a fold. The40000 GPU-s per-fold ceiling is a safeguard on **actual charged elapsed**, not a projection allowance and not a five-way resource reservation. The existing guard's conservative setup/termination safety margins, sampled RSS/VRAM/CPU/disk protections and global actual-training boundary remain; no extra budget is created. A total overrun or actual runaway stops/reports without extension or another continuation. + +## Observations and scope + +After each completed-epoch receipt, the coordinator records assigned GPU, currently visible GPU processes (UUID/PID/name/memory), own v2 worker identity and host load. The nvidia query has a5s timeout. Observation errors are retained as OBSERVATION_INCOMPLETE; this information never selects a device, changes the cost formula or attributes causality. Snapshot time is recorded; it is not continuous coverage of the preceding epoch. Snapshot elapsed time remains charged through the existing worker/guard elapsed ledger and is captured before constructing the new cost observation. No foreign process is stopped or modified. The prior epoch34 slowdown cause remains unproven, as Claude1794990899 accepted. + +New root `/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18` is unique and refuses existing preparation or started markers. All three generations are included in duplicate detection and retain shared GPU lock names. Source copies isolate the new logic from both immutable failed runs; worker and guard differ only in import/child-entry names. Original scientific comparison SHA remainscba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493. + +Twelve CPU synthetic checks passed first attempt: original science/guard/helper invariance, original projection formula, all generations detected, old ledger exact once, initial reserve, odd/even started medians, completed actual replacement, active-unmeasured reserve, total exceed despite safe actual, projected>40000 allowed when total fits, actual>40000 rejected even when total fits, and bounded observational snapshots/error preservation. No actual GPU query, data/model arrays or training was run by these tests. Log `artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log`. + +Status: implementation/contract for new narrow MeltAssist logic-and-ledger review, not launched. Actual new review must bind implementation commit, contract SHA, proof SHA, unchanged comparison SHA and full code map. Old PASS cannot substitute. The present handoff is complete when the binding is posted for review; it does not claim a review verdict, launch, scientific result, prediction/scoring or downstream item1b gate. diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json new file mode 100644 index 000000000..2f5d61fec --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json @@ -0,0 +1,2866 @@ +{ + "version": "seed1-training-comparison-v1", + "source_spec_sha256": "5dce5d0feb2b5cdd2703f53f0030b8127915d3464c7fa37e16507303b9c52b0d", + "scope": "five fresh seed1 grouped folds, original seed0 reused only in later separately reviewed postfit", + "folds": [ + { + "fold": 0, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000658_a40da330321b44f286a27361_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000190_185ecbc783f34408aef52119_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001107_78566795c3144719804f8d34_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000" + ], + "test": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001034_085828cd621d404a90ccd883_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "c": 1.026602777749673, + "clock_sha256": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json", + "learning_nodes": 8742060, + "learning_edges": 115060278, + "validation_nodes": 3255106, + "validation_edges": 42784756, + "test_nodes": 3125278, + "test_edges": 41266846 + }, + { + "fold": 1, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000796_dec80701adc84239b485449d_step_002", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001107_78566795c3144719804f8d34_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000190_185ecbc783f34408aef52119_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000" + ], + "test": [ + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00001040_74c42207fa764831b03f7658_step_000", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001300_b9951da825954791b59b979d_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001348_318b81a8e412438183201be7_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005" + ], + "c": 1.0302015087964238, + "clock_sha256": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef", + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json", + "learning_nodes": 9731293, + "learning_edges": 128102636, + "validation_nodes": 2819493, + "validation_edges": 37189406, + "test_nodes": 2571658, + "test_edges": 33819838 + }, + { + "fold": 2, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000391_fd490b564a44413884fab719_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001034_085828cd621d404a90ccd883_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000190_185ecbc783f34408aef52119_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000796_dec80701adc84239b485449d_step_002", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "test": [ + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000696_32309198fce9481896bd0a99_step_000", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001107_78566795c3144719804f8d34_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001586_763da6a559544098acbf4154_step_002" + ], + "c": 1.0257851343108513, + "clock_sha256": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911", + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json", + "learning_nodes": 9875407, + "learning_edges": 130004422, + "validation_nodes": 2573270, + "validation_edges": 33807616, + "test_nodes": 2673767, + "test_edges": 35299842 + }, + { + "fold": 3, + "learning": [ + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001040_74c42207fa764831b03f7658_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001107_78566795c3144719804f8d34_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000190_185ecbc783f34408aef52119_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000796_dec80701adc84239b485449d_step_002", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001300_b9951da825954791b59b979d_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000" + ], + "test": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "c": 1.0272648651548586, + "clock_sha256": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221", + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json", + "learning_nodes": 8118914, + "learning_edges": 106967636, + "validation_nodes": 3684752, + "validation_edges": 48502480, + "test_nodes": 3318778, + "test_edges": 43641764 + }, + { + "fold": 4, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000632_4f322b34e87c43289916c298_step_001", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001040_74c42207fa764831b03f7658_step_000", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001107_78566795c3144719804f8d34_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005" + ], + "validation": [ + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000837_b5409201f9a5495fa169af85_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001146_274866da478e4a009bed8750_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001586_763da6a559544098acbf4154_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "test": [ + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000190_185ecbc783f34408aef52119_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002" + ], + "c": 1.0300213434206609, + "clock_sha256": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f", + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json", + "learning_nodes": 9749420, + "learning_edges": 128488900, + "validation_nodes": 1940061, + "validation_edges": 25539390, + "test_nodes": 3432963, + "test_edges": 45083590 + } + ], + "seed": 1, + "epochs": 200, + "selection": "last_epoch_only", + "validation": "diagnostic only, original every10epochs", + "metric": "unchanged S2 query readout in later postfit", + "seed2": false, + "ensemble": false, + "original_S2_score_sha256": "c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45", + "original_S2_predictions_sha256": "3414a5ff8fd027a6529ea635b91e5966d5483ecd247f83f638490887f5a6946a" +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json new file mode 100644 index 000000000..176da7e52 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json @@ -0,0 +1,3449 @@ +{ + "version": "phase3-seed1-v2-total-projection", + "approval": "1794990750.013949", + "dispatch_approval": "1794983024.445549", + "seed": 1, + "epochs": 200, + "new_fits": 5, + "gpu_cap": 158717.68659210298, + "train_gpu_cap": 151517.68659210298, + "postfit_gpu_reserve": 7200, + "postfit_cpu_cap": 3600, + "phase3_gpu_cap": 612000, + "phase3_prior_gpu_once": 80903.62940728385, + "item1_recovery_root": "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18", + "track_a_training_root": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18", + "track_a_postfit_root": "/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18", + "pilot_contract": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json", + "jobs": [ + { + "id": 0, + "fold": 0, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000658_a40da330321b44f286a27361_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000190_185ecbc783f34408aef52119_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001107_78566795c3144719804f8d34_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000" + ], + "test": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001034_085828cd621d404a90ccd883_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "learning_nodes": 8742060, + "validation_nodes": 3255106, + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json", + "clock_sha256": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "c": 1.026602777749673, + "target_steps": 64000, + "learning_parts": 320, + "worker_cpus": [ + 0, + 1 + ], + "guard_cpus": [ + 10 + ], + "gpu_seconds": 40000, + "cpu_seconds": 40000, + "setup_gpu_seconds": 60, + "validation_seconds_bound": 12.383006267948076, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480 + }, + { + "id": 1, + "fold": 1, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000796_dec80701adc84239b485449d_step_002", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001107_78566795c3144719804f8d34_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000190_185ecbc783f34408aef52119_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000" + ], + "test": [ + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00001040_74c42207fa764831b03f7658_step_000", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001300_b9951da825954791b59b979d_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001348_318b81a8e412438183201be7_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005" + ], + "learning_nodes": 9731293, + "validation_nodes": 2819493, + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json", + "clock_sha256": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef", + "c": 1.0302015087964238, + "target_steps": 74800, + "learning_parts": 374, + "worker_cpus": [ + 2, + 3 + ], + "guard_cpus": [ + 11 + ], + "gpu_seconds": 40000, + "cpu_seconds": 40000, + "setup_gpu_seconds": 60, + "validation_seconds_bound": 11.502408961998299, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480 + }, + { + "id": 2, + "fold": 2, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000391_fd490b564a44413884fab719_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001034_085828cd621d404a90ccd883_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000190_185ecbc783f34408aef52119_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000796_dec80701adc84239b485449d_step_002", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "test": [ + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000696_32309198fce9481896bd0a99_step_000", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001107_78566795c3144719804f8d34_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001586_763da6a559544098acbf4154_step_002" + ], + "learning_nodes": 9875407, + "validation_nodes": 2573270, + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json", + "clock_sha256": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911", + "c": 1.0257851343108513, + "target_steps": 73200, + "learning_parts": 366, + "worker_cpus": [ + 4, + 5 + ], + "guard_cpus": [ + 12 + ], + "gpu_seconds": 40000, + "cpu_seconds": 40000, + "setup_gpu_seconds": 60, + "validation_seconds_bound": 9.668979812064208, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480 + }, + { + "id": 3, + "fold": 3, + "learning": [ + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001040_74c42207fa764831b03f7658_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001107_78566795c3144719804f8d34_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "validation": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000190_185ecbc783f34408aef52119_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000796_dec80701adc84239b485449d_step_002", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001300_b9951da825954791b59b979d_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000" + ], + "test": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "learning_nodes": 8118914, + "validation_nodes": 3684752, + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json", + "clock_sha256": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221", + "c": 1.0272648651548586, + "target_steps": 58200, + "learning_parts": 291, + "worker_cpus": [ + 6, + 7 + ], + "guard_cpus": [ + 13 + ], + "gpu_seconds": 40000, + "cpu_seconds": 40000, + "setup_gpu_seconds": 60, + "validation_seconds_bound": 14.438761183992028, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480 + }, + { + "id": 4, + "fold": 4, + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000049_aa208c45e8734584a147f595_step_000", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000198_826494e894454504b0971fd1_step_001", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000632_4f322b34e87c43289916c298_step_001", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001034_085828cd621d404a90ccd883_step_000", + "00001040_74c42207fa764831b03f7658_step_000", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001107_78566795c3144719804f8d34_step_000", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005" + ], + "validation": [ + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000837_b5409201f9a5495fa169af85_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001146_274866da478e4a009bed8750_step_001", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001586_763da6a559544098acbf4154_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "test": [ + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000190_185ecbc783f34408aef52119_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000658_a40da330321b44f286a27361_step_000", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000973_209426c5054945fab8a4af1b_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002" + ], + "learning_nodes": 9749420, + "validation_nodes": 1940061, + "clock_path": "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json", + "clock_sha256": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f", + "c": 1.0300213434206609, + "target_steps": 70000, + "learning_parts": 350, + "worker_cpus": [ + 8, + 9 + ], + "guard_cpus": [ + 14 + ], + "gpu_seconds": 40000, + "cpu_seconds": 40000, + "setup_gpu_seconds": 60, + "validation_seconds_bound": 7.502535316976719, + "rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 21474836480 + } + ], + "coordinator_cpus": [ + 14, + 15 + ], + "aggregate_rss_bytes": 68719476736, + "aggregate_disk_bytes": 107374182400, + "source_sha256": { + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/contract.json": "416183a09272714a179af790e5b7be85e5a92f1395afa6a5b9fe123a184b44b2", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-architecture-2026-11-17.md": "a15a13319d71b61c7677de41ab5bee2541d77009e0f8a9aab43e7ffccb0ea0c9", + "/home/frank14f/optane/MeltMind/references/2026-11-17/phase2-design/mold-h2-review-3541052d.md": "3541052da60cf4e79aab1611e2cc66cc4e97c630bdc6bc0b9a69df9bf672a460", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-execution-contract-2026-11-17.md": "1e74e1c0dcb19bb1c358ec13cb5346111eb3af7a0ef2ca3a11cb549e5da81d96", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-acceptance-2026-11-17/acceptance.json": "c8704f82239c1493a73b160cfa0db5b8ce7af4f3ebe5b164c04a377f6b137c8f", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/complete.json": "b1a2c509354cb0eb0f8a8c8ec9ec46d259b1a2f9a24072ffe6bd547f034cbe60", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s1-amendment-2026-11-17.md": "5c2f4a30faba1eb4edc8316d6fdd6b3172b20cea295a1eefc683075c9b94cf2f", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/contract.json": "45c8bcb1ddd2c8d73aa1dca65c0606a9fb4a515fd59c86786cf00f2622d791ec", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/complete.json": "01743041d16af0d35a8dbc43a3014ce8f935b8f8741430d242c14922a85fda56", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/post-terminal-cost.json": "d3d9d50db1dd37328f28c9801da5de11a2d8bc2873c71aaedb12d54bb7f60ffb", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/score/result.json": "82b3a77ea40ff63c694ec7ac170662816116c970bad2a13eee8faba84219662e", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-manifest.json": "fdaa61c80020f53af7f4a5c62c0ac24cf91063a7749918cc1937fa436b887e05", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/terminal-git-freeze.json": "40e998b70208684855f7387d3e521d9a73f7337dda241679d9c1f43453454585", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/terminal-authorization.json": "462efc91386f22e4614267383d73ac3c0c206ad49ffe6c68dfae0d73049a787f", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-git-freeze.json": "19d5cdf551d85f0b68e397a0defa904ef98c6bb1b42bbc16d1916a5832b5e609", + "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-authorization.json": "aeab9e49e80ccf5d86140a89466090275fae8d3edd720d58fcf2d2ea15e1766b", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756", + "/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/contract.json": "de4196e1c2fc242efd88e1218f3ec2166ab58f1c4d194e6ff1155c40d57d319c", + "/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/predictions-manifest.json": "66af9f9a9c4774358544ce04cf16ef7fb7f63db45335bb64378df664d7fbe51d", + "/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/score/result.json": "89ecf6d471cfd244377d7c069ea576a2817ab6188796276a00cf85709860508d", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/010.json": "9633a5ee7dbbb3f7d23ab9d695dc1701d926b904ab43106f29a356a5b18778ec", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/020.json": "58f0f65710b72a64be8cb69267ea7ea61d8933b5fcebb6705f2f85cf488a4fa5", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/030.json": "c0e7ac20ba42b37771fb512d8e361d0faec7c9d149dda107c700d9cfc00a198a", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/040.json": "6b9af61690ecc2f723dec91c84295a476a4f3e98424cf530de2e8d6637284805", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/050.json": "830ed3deee3037751f8997fb32b42e3157aceed8c9bfa7b315afc921264a585b", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/060.json": "5e77dcb1ad08a2ec94e8f9487b78ec5abf810b7fc6d41e04cfbbc5cfb22af230", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/070.json": "d72b7a9427efec8a804a1ce9e8160a01fe5956cb54612f4d42595d758e36ad90", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/080.json": "a3153d3f8a7a5302c38c0c09b197940d1eb79ce760cbd63dd0300aff60a9a657", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/090.json": "2ac2fde814b5d167d8312753d4827db3a824496c66e2c32fc236812ff603c44e", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/100.json": "7a9eaef91cf57cc592784a025561a24c85800842f75373e986f05afd09bee1c9", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/110.json": "132ae8718e286f6d736e8743c819749ad019bf22ef9f16911b801c95d205fad1", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/120.json": "8b37b513bc5b4d27329da2a1c31df47daf8cbf994e13b3893eba79e57e88a2dd", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/130.json": "16836051ea9c17c35fade1372c7d6d27f9a6d094450e0e6a586e9b07f7272569", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/140.json": "b2a48eb48652d13781cd138cfdac7d27baa7363be99fddb1d9be842489720cd2", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/150.json": "57fdbfce102a5bd08f37d8ce36188d1d7d2e911991b6286dca00762ef8c50f74", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/160.json": "d0eb813a7206f3f4cd12325d054ff297130232dd45a463150b7a7d92f4cc07f2", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/170.json": "685ed97210349304571117cc4b2fffb67a86181d356cf6f2b14b9e612efad144", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/180.json": "17e3d70a147de465a0273c799fc84366780c5cb5507b34ef315d7e8ab8ff05ca", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/190.json": "fda3852e9d771c67a9a41b73064fdd6a318b8d38d4ab5db3cbaabea9e3b2e402", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/200.json": "b6aa4d4e046a7440073074f027e7107de6fdccfeb8f1755d2d6b1bcce6a59128", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/contract.json": "76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/complete.json": "4cfecfbf5b9242ae08f5100a3482ff61b1dc01435b74363732501be6b2ea291e", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/complete.json": "2a55287167daed3ff541210f0fb3f32b8971b70976329cf29ca81828cbf06685", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/post-terminal-cost.json": "07f51a3e172b786ddcd843901581efe89e8bbcad31930bddb232f6430a034c15", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/score/result.json": "c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/predictions-manifest.json": "3414a5ff8fd027a6529ea635b91e5966d5483ecd247f83f638490887f5a6946a", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json": "0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/complete.json": "602e978eedf37b22208eb1dee4c7fb098b3d31c242d6db148a16cc2b9d2e8194", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json": "ca576e271715c862ee21e75b2dec7b03e45317e5f8bb073adae016a8001a0a59", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/terminal-git-freeze.json": "35dd7ea29c47f914efc0384cf2b2ca8768f867b50757d3bf5172add0e631da20", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/terminal-authorization.json": "16e73c59bbdf0fa49697e24e2cfd2bc0e637f5ba03b4d3113eccfc4f5e074588", + "/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-seed-sensitivity/mold-seed-ensemble-spec.md": "5dce5d0feb2b5cdd2703f53f0030b8127915d3464c7fa37e16507303b9c52b0d", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/010.json": "3a2933dac351221f27bd95e60a3c8d38d58a3b5446d26f305829f10f3d145069", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/020.json": "db7581756a99f193e80bc5adc480e7d1946c8aa3aeae216203cb5eed6d4ce82b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/030.json": "68269a3b259360de1118097271f9b84781730e3da27010c31d7b848727d752ea", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/040.json": "eed4dcb63fbf6fb7a248c66a2be37375881e6db9fa1bb35661720c1b8200f5f2", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/050.json": "3303cbe0101fcc01ba7654bbdcbf982451a0a2c430b1701809bc230da5582c77", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/060.json": "7d2992dbc8b3a543f40146af31bf2ef2beb93fb79557f689d4b5ff4f214f0969", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/070.json": "c18ec5fdc3cfe6878cc174da72ad163943f39925b6da10cf4783b5f562077dd6", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/080.json": "2b31cd1e0cbc19109bac2f4f44aa0fa37b1659196a96359a8e5fbd521031c85e", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/090.json": "0170b582e6b1f26711378d28ac82c52019fbbc9b75cd0a6d201831daf583fb00", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/100.json": "4211943eb968b09aec767235eeb296e7ae24b80d5b508a4d0a383bf8ec285a3f", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/110.json": "b5548033e8ed99fd5d857a3715c1238af85d2008b6948089047171162d0c477a", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/120.json": "6a42e0041510a722553e5949f37bb053afaa8ca258862cc80aa774bab61187e4", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/130.json": "0dc7943f8a01b3bf09155ad22e7e3d2465f9f42e37761693dce5afc1be0668fd", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/140.json": "b987e046ff5ab8df1f57b1df1906355ebb8c59bebfd636c6482b787228bda21b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/150.json": "fe515408a6d4316ab634db9c2cb9c494f866a3f9b37d11b20843aad2392cff40", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/160.json": "955f537198e434305a74cc2bbf920bb7a1334346db7671e0319dc64a774e6f9a", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/170.json": "ee620d48cff4e35a38f4279c2ce9414b40d613d7d71af46c3bd353541b97e775", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/180.json": "7bf55c95e2c07e16f51ea91b3147750a8eb6d67c567b167b3a5040fb9fca1d3b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/190.json": "9487d038e82aa6fc6afb1fa2314781bca1f926f69683ddc1bbd87ef6beb6ada3", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/1/worker/history/200.json": "1f84ff2a3fccb83abbc2974076967b2c2cde5f306155e2ff97d1697911018074", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/010.json": "660be02c260d20966e339b10eb0bc7a3563073669a47c85a48d122d296da86e6", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/020.json": "245056d6ad0a647949c08853772028242697f8d2a8f78a85552653e722dc70ff", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/030.json": "04c3057395f7bd036969a613d44f249dd023e87c459bd1836c29f8d00c6032e3", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/040.json": "bb86a02e96a25de992184afb04bb4c4257f4d7b5a382d39ac446c7a0cc3fe092", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/050.json": "1e414fe371f3c0df069a850d935b5b7137a56a66bf0a9ee56f5d52020659c946", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/060.json": "e579a62f9a5a847a1e6041e1e84605cee60a804af408ecb8a89781372e233585", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/070.json": "b188c6cdc3a611a8107264d708d5f582efcf39156ad606c48f4816c7c9ba3cec", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/080.json": "0ce4d84d5f4c5781128274c091f49d2002e897becbab40ffe735123f67a4a893", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/090.json": "9a5c8e8762c61b856fc1baf71909cc8668abb9454b8519e474c8451cc00e05c7", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/100.json": "500a79841ff67c9ed88963e771c0155fc80dfdbc64d3baac0352599f99a5633d", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/110.json": "1e7c9d4400182376288ef518c587ab2f07e021fe1b582ea63d507ce48b40e112", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/120.json": "b70796ad458a5d93a2b1b29b8b6f4575a1216cce2faa733ea278e7f11b488f8e", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/130.json": "b18b09584325f955f6a84f58a37674049edd3f6f21377bee08488750cabb4e8c", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/140.json": "c7d02cc4336d8cb6d1411c6f7acf699156f700fa12908515e05a0958ad9a82aa", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/150.json": "79cbd5cd4cd6991ddeb4ac3ff18058405ad178b8e433e1d3606539ef36ebd87d", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/160.json": "f2ab6f3bd3b87fe4cd5a556d11c6683ad48460ad3b92af9b5c919097511b794b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/170.json": "6387a34206cd193f95f8efbb17cb4b44496b3eb47e1c2eed0ee7aa7601a21299", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/180.json": "cdcf7d045735553da202992b1f31b44f516cdafd403fd9a10e11d41f6c874429", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/190.json": "22f2fac1df97e625b35ae322b6de79234368d6e62752ef2b987c67e5c3dc5c21", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/2/worker/history/200.json": "fdc542244ad3640785396a32f75d183aee2cef6c65557290d04a2ad3a5d31ee2", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/010.json": "47b0852b172cc7b9b7dfd7f491f9af3587b766a84a94eacbe85436fe65d93b40", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/020.json": "48aea6009f3ef91798e1f349fa7e6b958416d9812bfe471026cf06929f1c4f16", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/030.json": "23e5e6acecb81a7bca395a390cd0271856e63932bc7976478dc9f5d9cb0f58ba", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/040.json": "7dfc0c954d3eb69edc87509298469cb5644b6da34bb05add35ce9be5ce274bd3", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/050.json": "f7b66507973356fa152f93689fc1ad89e36d9409ae3c1e53eb83cf731ee50230", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/060.json": "89049970b4537198bdee1740d58b872fef5cdef193a5350f4ce5c2b33c9120fb", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/070.json": "e7c59ceb7d9e327309937eff05b1f4d7aeaba1fc84f645349b5f9119bc518e85", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/080.json": "0777905db7421e5d6736a9918504a55899e38eacaa3e99363f746df1e54aef86", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/090.json": "b921f09643c77367887949a5e8db6eea8f5a182db8f2d79f066c4d824852ebf8", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/100.json": "533d01cde83a0fcea761aa10b5b55eea2d18a7f05cd4d30403e650dd367c98e6", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/110.json": "c201d6a1d51cba925f71b9af9a1f50329a0e89cee57663b7cb62b438432dfe08", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/120.json": "09843cb12ab778aad4ce7cb18f02a519c283f7b26c3fc20466fa0c1d65e519b5", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/130.json": "c850f9b9ef258032593cf49a52e8017edf9f053e619b50213551f2dcd64a9216", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/140.json": "26459cbac53ecba6110bd60904f285d5b10a0b6cf74930dfe98c93cff6f5078b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/150.json": "01417c763e47b539a514c2c38ae6252996ae9173cc7f4200320a026eca0f7996", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/160.json": "0b45bfa142a63f7320156ec4762f02b669fa60dc5e4b887ab4ef15e35fbfc3c6", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/170.json": "f5f8f02b56c3226469670787d075a0042884e2a1c8221c55473b97706106b8f3", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/180.json": "be735ad8ce926e816ebd4c466719614cb74748e560002ade2c77ed2e3e2ac8ed", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/190.json": "0fcb44766454ddfa5264bbb861e01c2c050676f09fc960e2f20ac88783028eba", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/3/worker/history/200.json": "414ded64f5d27afdbfac7afd67df0b662ea776f4db0d0cfc7eb36e4bc0f708f7", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/010.json": "aed83afaed4876979bc9dcc1d2e97c1707dc88ca62f97ac130a48c10a92ff962", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/020.json": "5ed8656f7c74b08814624b44ed0f00e960290cde940c3b00e594aa07864ea8de", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/030.json": "04c5054587e35df5b791b9a3ed9d31156a51c2331334e398c6ec911897e5a4cf", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/040.json": "dd97fb8003ee1fe9a2585f925c4a5136863a853da5bddd00c68617102f5a2174", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/050.json": "4988d0b55ec5805940079fa1706e5ad5bb8036ac8ceefa308f884ed88ddcee36", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/060.json": "10000cb2dc20d81b0121b42b1af069b5f3fb5bcf4e720a250d975427c8318b00", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/070.json": "3b74ff894591d6b110a1f66093bd67ae71df48d115d663c776d1b305363b553e", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/080.json": "01155692da2a64a2541339b95fb8ea2e79a284f1834bb7f9c80acc4da1f06d48", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/090.json": "911210d53f2d95df1527c988cb4d6ad905db72272be5b3c7d74115ff12a12bba", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/100.json": "3c58cff7d74a9892af09b5cf003bd2c5ee49c8d3c9ccbdda5a6820176ab74dce", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/110.json": "955bf882c4612cab154c503cbe317691567e8ca503eea34f99d24d09ec5fb1b3", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/120.json": "8513761dedbb1c4b0aea1cb1a1cb8c71e5b02068cd75b44118a3fa7b19957eab", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/130.json": "55f6d092bad1656df032a1b380f233b454486a96dfd2cc1465ac7bc1ad347655", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/140.json": "0c61ad235001d4bf32709ccfb3653d6e3a7a3efa0c9480685f3e93b1e3564993", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/150.json": "c7f0f9ea10f42762dc842967ebe3a0a53ac7097fb63d432afb804b15956ed830", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/160.json": "f6e6334eb76a5cd91232090045d5ae2bdf7e9649a133cd9b79ffe1812ad96d26", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/170.json": "fc05e8386c25780119bb5dea8bce7d3a810c518d1d54d362742358e844ba2387", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/180.json": "4eaaf9825c95ebbe48f07601999de18853434fed7fbe8da882e791de6ba97a9c", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/190.json": "1d43dd816d41a6db9a2d2c19b75d713cfc075e392fe680b429f75ce63ba5d063", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/folds/4/worker/history/200.json": "0a1adb8ce48ad6d31c6bfb459c1084fb76cf98d25f9e18a6cbb09db76d59d571", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/review.json": "79b5554654f8bfe5a06be49bf7af5cfe7d6cf467bdf55f53994b342e00545874", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/actual-start-audit.json": "ddbf83882e5f272366cc7d987dc57f6f4a26ca4fdd2fb72a75009e4e90d7dab1", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/implementation-git-freeze.json": "04592d0fd72cc8a71201e5dea145bc8c618ff05d15a78b257f0d2dc60ab54431", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/launch-attempt.json": "70630ac4d999f97709f6de3692f131e224c9a4fddcf001d8cfbc849463eeb759", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/implementation-review-authorization.json": "4486c7aaf6d17ea7ef387a70f6f4d89172c73b3487f4f1cc2b153b5de8f1d6c9", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/global-stop.json": "67fd96999a906517c9ed79761c5dd03ae776e6e1bb5af2f25e76917b0cc08c30", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/live-progress.json": "d2fdbb6eae78242f331a184bd16755d2f35f3da23e03fd9b2a92cde40434bd3b", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/dispatch-idle-observation.json": "d4cea9d15d069204d2d3441b253b165fdb23fc07d07c9b7a7ac76ac7058db212", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/implementation-authorization.json": "0febadcd5d8a361d2db828659707e0de01108d0ed8d401e361f976f66dcbf07c", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/started.json": "01ce0f0568f377bd9ab1c0ccc8a69391473def0208357b42f71c95f66c34360f", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/implementation-review-git-freeze.json": "b96f04681772fc5197618603099844396e5b9a3f41d2ba746363e665d047ca20", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/first-two-epoch-audit.json": "b400ed6f5c55f46956b5113c6b27f9ee9c4a8287426100896a20ca98c3d106ce", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/comparison-contract.json": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/service.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/review-adoption.json": "cccaf3c09aba2111836eafa8b2c97a8e66392c55e0d66a982d6b3895e8a9e22d", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/expected-review-binding.json": "587613901b11843d088377abd7c839dc108ada8aad412b0e619c1938b9b985f6", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/complete.json": "1777937a01df8e39d2754181f83634227507cd57eb04847fce14072e183ae1fe", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/service-launch.json": "c2f7c6da32563f3465867083b9d2173856ecc152b762f8df0542c2fc47d31b42", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/latest-cost-projection.json": "1484d69cad773f6329a8a16668dc0836269951a511d72d73dd2cd8acb3d301bd", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/contract.json": "95d23536a653ef2d2598127cabe29c3b4bcab40bcc0f2fafee0a2931893395ff", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/host-precheck.json": "711802b452c82e002bcccfbf5d47c5c80d5767a884106fe8b5f344ff12d07a7b", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/terminal-ledger.json": "8ff109879bf67a5d9f71453c0874de550666ad6b2226c58839c455478bfcd2fc", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/preparation-audit.json": "2c8921b55ee6dfac85d63ed712f7007d8e4d1ae76eebe9c73a1f62325db46f00", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/first-epoch-projection.json": "62f8b0ed3047db18e41e30e7b47a0c5743c71aa9b4849e182089f4041825a462", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/dispatch.json": "994099d57c80b9086ba41b7ba8a5f4379a8c1233cce6095b52561a91a4adf7df", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/live-progress.json": "84d30ab9481e0c34a26ecd66646a8ea0495c9d9ac408b020d3ea6dc73c0533a4", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/assignment.json": "bac79bb68cc7ae29209d9039342fc5fa7e95a538a9115758f5e4f4b6a06422b7", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/started.json": "6d0a8d6c1c926f213811bef1e2d715de288cd536299825b4bc410fbb4ad90966", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/entry.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/complete.json": "dab4f4fc422552efe610021c28366ed544549e9a58ee20b32061077c8e0ff8e6", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/idle-polls.json": "2c3f73e8b046f5275eb08499e1dfec9bf82cf1f227e9e89c2479b6c41334c6af", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/latest-cost-projection.json": "b6cc47ff2ba67fa22163bdaa7f81c2539fa8f8d128ea87643c7d960963f2183c", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/guards/train.started.json": "b9341a0b9af345ef3a3947138fe0a44cc8ab32a42bd19c5eca010b2ff4bcdb19", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/guards/train.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/guards/ledger.json": "6c40fcdc0ad462f4bbb1dad761647cb047c72a01dbf18ee61a58cf58869f8e21", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/guards/train.receipt.json": "dda0b95191d72c5742b187fd9d6c87321ac45c0a7bbb1cef4580c01c9007562f", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/progress.json": "e668070c67d7743d4d0ba8813fa040d419a50e13f961fdc1dcc5a677b6bea2d0", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/learning-target-access.jsonl": "81ded05f264b3081c47be1875b478c7691a152934c18a58fdaf3ac867a783105", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/standardization.json": "bd7f0fc755a37bd08965aa7bf630ee4a3f2e036276789bf70346855f3f1d3d70", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/fold-identity.json": "4c673534c224d291039989db84d0b8ec673230ee967a8da16085b6c995089f4f", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/environment.json": "ad6ac7cff518aa63e044d8d98c27522085d3f695617ca1e67245734f55eb9c85", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/history.jsonl": "1aadfb1c503766e4597db26c9995a8405454c5da690d97bb19999bf94cec7bdc", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/history/002.json": "d915bc0ab61466fb686701af0791096f122f584740c44832c3ca90b17b5f14ed", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/history/001.json": "619a9cbdfd627122dda39b171828e310572e74e58eaf2806a8dec79bc7112ad5", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/normalization/progress.json": "c5c0b4b842611be630434b935d542f8ed07916d146fdca10c6cdab0107bcdb82", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/0/worker/normalization/standardization.json": "bd7f0fc755a37bd08965aa7bf630ee4a3f2e036276789bf70346855f3f1d3d70", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/first-epoch-projection.json": "f00b246c47864fd7f4fab7c47cbe7b5b17240ed52ceff7cd07374194c003ebea", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/dispatch.json": "9d60c5f884c7c98c2b49b48d0b3a62f4be6efc4687b42419980628c31c3e9e0a", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/live-progress.json": "b0aac96bcc40b185a4890d8b4816293926fd4ca43e5e1ae7852ca4bf21f970ba", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/assignment.json": "2e59482d320e81f4986fa3bacc9a2c250055d24460f2cc7ee6789ca40987c788", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/started.json": "5da87e3d467496b117cd6c59d0820fa234c67c3b8bbc2526b3d891257fbcf56f", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/entry.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/complete.json": "59e4e71b1fa7c0b1b37732a02d202aeb83451afed1fad8ef37d4e4185e8c148e", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/idle-polls.json": "130489ffa99132bc3b2e0a63ebecf67897bd259f8b8ac17e1376618202b9094a", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/latest-cost-projection.json": "1484d69cad773f6329a8a16668dc0836269951a511d72d73dd2cd8acb3d301bd", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/guards/train.started.json": "d37eb6e2951e16e56f49638afb494f523701a122274d4f6d9dbc68cc1bd3ad7a", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/guards/train.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/guards/ledger.json": "0464085b1e89dee685c07470a11ceb6e27a0f11c5eda80383e623b9aab2eb2ab", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/guards/train.receipt.json": "62b1ec4bca98554cc47072755e4421496a1995ac85d1da6ca9a6718ba67e3d81", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/progress.json": "e6bc07751b87f50460066a27fbcff865d13f8d6ccd9cf2fb86afaafe88382f54", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/learning-target-access.jsonl": "24b59a49e14ae04a497ba14265e7ee88018a465aeb5d35256255b4876253cae0", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/standardization.json": "3c380b394d51c805d89d97885025f47ee8d157c81197edc108ec56d334d97fa8", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/fold-identity.json": "1f5c96df661f06943cb0ff69f4c7a420bef85b43c5cd8a93a8c430908bda5358", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/environment.json": "3a1ac817601927098d37be560eb17cc9bf4db16ec5bf57a9dfebdbbc59bb7671", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/history.jsonl": "4a8a8864ea24c9575e5f7fb01ed666c02bf39bfc88b4e0d4c0d61a2e20592937", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/history/002.json": "e3df06de6fe41ea7e76b2b15d477c3b8aa3433ced9a874e49f87d3063d72f5a7", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/history/001.json": "5389a807cb92606d43f0d5fde7ed0bf4f2dbf661cefd80eb00f1701f19426711", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/normalization/progress.json": "e50a7f6f2d420153f45b850430fa0bc48d99dd56fb6e792b827c59df16792927", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/jobs/1/worker/normalization/standardization.json": "3c380b394d51c805d89d97885025f47ee8d157c81197edc108ec56d334d97fa8", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/budget-stop-audit.json": "c1457c69c7b58ddc82470da000170a489f7d9acd8318d8e109282e03ce103a55", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/budget-stop-git-freeze.json": "da1bbafe87bde23344aaae18152b2315eb333249a8ee5556fe386867eff8cd9d", + "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18/budget-stop-authorization.json": "218e18c57bc45d6f8a7926a1c21437c641cf26feca72a2bfca47d2f68b652f0e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/review.json": "5c8359c9ef608b11cc2cac81a7c433b1cec79a16f31fd457e28636bdd0616c26", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/actual-start-audit.json": "05b21e073be63f82093cac067c694916d8906e888227220fc6cf62b157bf369d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/implementation-git-freeze.json": "9188a5fd7c2b26e45018b1901429a7660f8edc46628365937ac234a9c065b8b8", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/launch-attempt.json": "492854cb329ef4cee981feecf390074d04240c639adaede21fd72db8f86faf82", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/implementation-review-authorization.json": "00f1befe65bc54e411cd65ec9ad059a44b810016397b46bdb91b2f69182dacb7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/global-stop.json": "02fb76ed0103dff328498a98fca64c79588268734b11878a70d6caae3763cdfc", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/live-progress.json": "360dedf475cc71074402c5bbe29230258144c48f729e59dce247bc0df5444a23", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/dispatch-idle-observation.json": "3806f33c5f3685ac1090c9f79161b3ff34aafec472085fe82fe73888aac4b1f1", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/implementation-authorization.json": "5730f360704162c6a9090748f61e0d36705d2bedced20ffc1acb0ad758629bcc", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/started.json": "46c845885bb53b496f866a376d4838dedaecc2a147d0c79b852a017c157c12e8", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/implementation-review-git-freeze.json": "3dbd1a60f4c4dcaa2af836a7edcafa588067bdc5e7197b6a7c6367083fa82e73", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/comparison-contract.json": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/service.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/first-dispatched-epoch-audit.json": "6c15f65d9cff1609fab8a2c9ec1cdbe91f522e334f7c0e5d949ac4c5e931f5c5", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/review-adoption.json": "ac3f7f655cd4d4c89c47e3cc98dad7a45b14a5eeaa65a71a18c82b10698e0729", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/expected-review-binding.json": "eec01a5ede67369da6514ce9a7f1000e41715f8e2f73499bd587d9696b58a3f9", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/complete.json": "853bb37af011316d5bd0a4d8d60c4100e46b89c804ef4fe0af26becf3082a550", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/service-launch.json": "9c8002194d0cabcd48045fd6a26b9f73db74df9b9c102a3890fd140fcaf785be", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/latest-cost-projection.json": "db62385065f4cad593337af343c218d852a4d9c7244ea5cf763f21d08759a6f0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/contract.json": "e90bc1b5e3781b22d2aed4b7a9b4f1f6b64ca0eebdcb1c6fa7eb42696d9752b0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/host-precheck.json": "a1b7ed56260fda115a3240a5be3b388a56f40c534052342adc50fa233c943aa4", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/terminal-ledger.json": "ed55d8425c6d65a38dc7725d77ee3356ed623676af4f80cb652c024cd8575033", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/preparation-audit.json": "5579b033d1d9659aa33c055d870d2b0db922d64298da95f0277c8a74c258465f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/first-epoch-projection.json": "7abc59b28f3e82df23e9d5ee22e28f1f5177152d8c340710d7b945637b09d669", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/dispatch.json": "416281ff537b33d3d8c153ae23119f9a2783855bf5121b106021dcd178392df1", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/live-progress.json": "295219e0ffae9b0608b6fde64b09d25b61b1c67a7f0a101f977f973303ffb51f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/assignment.json": "b0c947264e73b10bd9fce3dc5f6c6e9ef3c0bfae257233eca832e7b083f1bf85", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/started.json": "af39c5c3680b8afb74f5603cbff9ecd48aee1419cafac8efb43aee6494fb0414", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/entry.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/complete.json": "3ebdda6a50efbf993ef40ba9af0461173f2acd4f270382d6f3428aad8d16d64b", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/idle-polls.json": "5f496f981a7d477b28eeff598dff456cb069c37b34349a01519abacb24bbd3d7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/latest-cost-projection.json": "22207bf0323f639c2dfb6522a94a4beca2ac54d4cc3b9ee82c0eabbe469abd55", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/guards/train.started.json": "e436d814a9c81803e223e494ec44a159636b976d8d730aa089d4ce43f1a2b7be", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/guards/train.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/guards/ledger.json": "1a426d404e8df51c5ee01b680923b9bb7f4904c33953fda1a6aadbd0133c9654", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/guards/train.receipt.json": "02a5f02269a37a5d5d2f510ab478ea975b0578cbcec0f9366b214d961c8463e4", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/progress.json": "651195f88440b4eacce284753bc3ac1304738082cf97b6f945901b0dcfe1409e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/learning-target-access.jsonl": "68f8187de0cb3866e04cc53c52c44fa3c3c243798cf48ff461afdc4a980c38f0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/standardization.json": "c5e00af75f76ef8e0833450032fce4132d7a6641c54fb256e06889e935924fc4", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/fold-identity.json": "4c673534c224d291039989db84d0b8ec673230ee967a8da16085b6c995089f4f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/environment.json": "ad6ac7cff518aa63e044d8d98c27522085d3f695617ca1e67245734f55eb9c85", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history.jsonl": "4079ff01b2155b9248c6afe949926e342e11a0e228cc673c3bb7ab610490795c", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/validation-target-access.jsonl": "424d9cabab1f2d69d2e531c48af724e6be8b97901802d020572db0c3fdbc6d5b", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/031.json": "8055a4c774762fdc45420ba70a7ef03a8692177c6b8858d4e6178fa0984b2f72", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/038.json": "fa16aa0c866e321dbe6933ba185fbf6d60480563f6bcb72b91ef07793672e7ea", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/015.json": "82fbcc0ddc40181eb1a106549f6bc264e098d8036e325b0a0fba38752790b57f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/006.json": "577df0bbf384272aec171d7be03676664a7370be29715d7a96091382347daffa", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/027.json": "1d2d5779fbdee880d892a83b93b1110b712e3d7729f4ef60907c53110703ccdd", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/024.json": "ea8872d28146cf80a613452c3f979df95dcc8cb0a6acee90f34e70712db39cbe", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/044.json": "df80a0be56e0be25c00118fb9867e77d16760e42ac0972ca76cea4c706465d92", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/023.json": "cfbfc49d5f0038018edd09a76301266cd8bcff6c37acbcb5e6a4e882dcd78d35", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/003.json": "e9286529178e74c05514eb518e830de53c76e3414f800d941eace2a763354915", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/028.json": "6c21cfb91a3d8961d7fb43bc08b2e0ca7415cd8343f3e1be37904b12a2cc4893", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/017.json": "5774f78255e7841fd09884f83da7a54b9e75cf730565703b279d498f1f02204d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/034.json": "5b27c7e537ed46f2d436d08c4ead8da485f2cf4b008e506dde7d52427fcd873d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/035.json": "e1500ae1d7852855c90b06e4e333e208c569b24e61b37ecdf7ed6f7e101a43ef", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/036.json": "6f4d03aad7e206a57d893a0e2955a121042eeb7920269ae54b3ff725f2ec2fb7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/039.json": "d20683776e2d3372563775296511454d50ec286f0f8e6bb4d4c178ee1a17a863", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/019.json": "b3a4f7d369e5ba8d1a386a9cc887cb11d71a6378772338961529d7174bcf9178", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/008.json": "1d4c914cf6e3eb9fd460d8d90fc3ffa1152075c8be435fca748429220994cea7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/013.json": "5b2ed4cca4bcde57be713b67f750488dd45bc33e14b6718771d5bb32d4882c07", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/020.json": "da36ecbc661922f9a055e7e3ac8af308c0ffb1ff01081783012728f2935ea643", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/007.json": "e90ab8d37e209bd9f6a02e52f807f24efe18b55403488184805b453ef6078eb2", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/032.json": "d3ee2d5cd6a156b5ce1b5880c5b9cd9b9f7317d622a37fb267ceebeafb5ef6f7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/018.json": "84011f1ded3f98d910b6a24faeafe9aa81de932d6ab7b5e43db6e8eb8df6c355", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/025.json": "964c3fe71fc6514fdcdadd8296c779e9703a92f0fe24904124e9a198256257ea", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/033.json": "031bb8848d6b41e3096d8f440ae546314dd279b438cb400c53df43f2c6a62ec3", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/014.json": "a8b2fb7ef6d2c7ed3b4a23da8ca851b6f3a3d52737b2946363c63261e7944d7e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/005.json": "0e543f024c026aa7b4c10f4921773934ef16811773465accf9bf717131cf28a7", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/002.json": "bfbab4ae0f70146cea7880d9ec3f9840a834e9e109d02ecaa6addbfc6bac390d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/004.json": "4f0242ac35770aaffdcb7e0df7cc49c313fd8a9368599d79ebcd03c8ebb96d5a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/042.json": "df806d01e25f1b9f62e7232e929f7c2f4b644de737a61ec807371ec9f60733f5", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/011.json": "cdf1fbf644baa1c5f05b249608037ba461baf9c6ac8a613dd44a870413a2463d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/030.json": "a970619f130ea24a502bcb1f58ed5175055188a11cb3e5b5b17f0e0980234bad", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/022.json": "47da23dca136271a29831562129de0757b40950809591a66a95f760d272cc1a9", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/016.json": "eb5cf0b81a49083bdc31916eed48004894b7f2d1bcc68b572bc372fd54233b96", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/010.json": "287dc4b4858b0e1da5a9598511bbaf7d3a4991fe1035c4540bf387c6554ab06e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/001.json": "13511f217ce2dd49078b16e1daaa5e6e0462f555157299d299ef403444538310", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/046.json": "3fc683c080b5919875adf468d6e22f96cb6686730e5356151706407b17909175", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/021.json": "ff714a28a5d5f86ea1d6594e282cfdb73a38b221df82fd6c904045bf8fc8eb99", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/045.json": "06095f7e2ed5c8d067a7e8399f75baee1440a5cf28c868a832f04eb057ddbcc1", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/009.json": "b8a6db8d95a1000518f73dbb26fb659ee7b8171a635ddbcf2c629d9a92f1fc17", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/040.json": "8c8ec858b5bfffa6b5d6224cb4f319baf194fe9dabe9c856192edb9e375e55bc", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/037.json": "66c6164a5a65ea19ff1d0242bd0106cb4bc99a832b4a87f5f041701547c346b2", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/012.json": "d90e2f34c470c89acfac133702e6b7f9e000ca64d24b0022a303beebfe66fc3e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/029.json": "9c826964ccbab60291d7ebd79af79681eb28b0a7a9cad6b055d788d6ec3bc285", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/043.json": "62851ca02dbdd6d1a7db4070009bce5bf6243663e5e8bfecc396442f2db2efcc", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/026.json": "9040fed3dc16894911fabbe2c76137913c09524e5d7b3289e579a18514f79d35", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/history/041.json": "0f11fe526d3ecab74716979e24fc5f9315ce946d38bfd7621e90b8ca7d0ff485", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/normalization/progress.json": "e3a7fa01a3476c5774ec20366a0442bbc9365008b2c51ce73fa755d20eccc9f2", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/0/worker/normalization/standardization.json": "c5e00af75f76ef8e0833450032fce4132d7a6641c54fb256e06889e935924fc4", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/first-epoch-projection.json": "70b8c51e2b23681e878551363e6bc4fb7a5adce67bf6a265a51a75a287fce46a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/dispatch.json": "6d98ad6320fd1269add3fc3f7cfd53b230f3c43ab4e78aaaadc8c8071c60d0e4", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/live-progress.json": "a7dec1cc84816a3b114411236e2a6c3189ecdfd7f434f348b0eea44ecc2dcc40", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/assignment.json": "a1d0890a4e14964c31ce5738b312b25041e21e3514b19d3758a9d2010e74b8a5", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/started.json": "9ba16ec35aa87bdf71813d01419bd440ac1c2517198c3b50aa6c9d90b9bd66dc", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/entry.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/complete.json": "d95c159b3285858747b894106f0a434729e0829fd150d91aa12003dc5731f5ce", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/idle-polls.json": "44278762da17fec37f280b15ea72475c039a3852c065ce250d15cadc6ad5a0c9", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/latest-cost-projection.json": "db62385065f4cad593337af343c218d852a4d9c7244ea5cf763f21d08759a6f0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/guards/train.started.json": "934b0376da6bd3df5afaf49a41a9e83d4baf9ece28f2417725ba4fc7dcf36a8a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/guards/train.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/guards/ledger.json": "6f8cb286b7a923113175a63fae7ddf76c310da86ea791e564f6235e464312f26", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/guards/train.receipt.json": "aff080a7166a0059ec7e75260742ddebfeeff6d7d6e17fc3525551e258cd8429", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/progress.json": "541280728baf5939b6b6c9ce01d0b4cfe046da9e964554d0b3689c4d7c608459", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/learning-target-access.jsonl": "c0c4616548d381158961394eac60eb6b9f95873d65b33748193ba4b1336412c8", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/standardization.json": "a6ca68193afaed449f42486d4fc1e42838a63ed4c3a9c2083771bb3154297aee", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/fold-identity.json": "1f5c96df661f06943cb0ff69f4c7a420bef85b43c5cd8a93a8c430908bda5358", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/environment.json": "3a1ac817601927098d37be560eb17cc9bf4db16ec5bf57a9dfebdbbc59bb7671", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history.jsonl": "e2f763579f1b1eeed55024dbdbcc521ed4741ba972ed98fbd98d75bca9a9cb65", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/validation-target-access.jsonl": "1b18ed2d2ae7e6153f761cc733fbf3650e4790607b24b51c2ebcdf23d263c837", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/031.json": "b85a130a55f010d66c912ed58190eae45447c84e8e1b1133a5b6409c1905b58a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/015.json": "bd73835d9d2d1d1d20f605a7f78f699396b21f6d6317852cd93592cc49e38f4b", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/006.json": "0663ce09635937e329cbc3ff62521120d30c3bcb400377974c255fb52392443f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/027.json": "4ca56975597af445997eb0b7b4e1621ee483c5bdc4bf2baf30f26bf93454aa3a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/024.json": "61e6eddfcbaf74f08f12558cd111c0cdee518ee457bc43cffdf3a8b62491f1dd", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/023.json": "bd6f65aac6fec9566d12ccbe976914abbba7a427282517d539685726204e75ab", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/003.json": "9d9a5861aa89a5dd19b14b0ea7d63c677787f91bc1a7b58f814e8e5a45637ace", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/028.json": "7f77ec08d50267df5401907e7d1442f7f2165c306ec88c7beced206fd39f29a0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/017.json": "af3ae3d13c65a92e7a495befa12a513262da05b15dac2835fe936b858d089c06", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/034.json": "f286d6515f46fc5658a8131e9360b78145392386ba2dfe239ef79d8ca6784cd1", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/019.json": "a95346bb76b7081ccb89d538838240272246122dffb98ea6b9c202974157ec5d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/008.json": "9f9e32df13114ce3743d2a10cb9f63f1042c0d4ece0cb167764faaa603223a6a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/013.json": "3b0fec01ab7ceab10df6ea03c7970843ce835fd7affef8b20ee62050b2ad3c5d", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/020.json": "228a1798534e2024425589d9b0d0eb727c026565005e6ae0a54dfc3dcd6f03d0", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/007.json": "185f86253f20590fa4fa27f502a798b571635bc27138f9afe8dfc81c547e5384", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/032.json": "8744e3f58d8e64d154ae1813b959f372675197486f1c144b75b9ee6b9d2d6a05", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/018.json": "c843c2b880d230a8fd7280836729869ce02979e65c632c157c284598c44b8865", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/025.json": "4c8f9c6bb7797d8f7436f71b96e9bc47946675dfa4e4bd06206f53e1ef7e7c73", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/033.json": "a15e732baa6e853e13ce6491a2ad94b630f035f4f609c66d39cee724221eb8f3", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/014.json": "e5c94b20c39d148b238bc9a5cc88e52650ec8905a329e502f60c946815d03554", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/005.json": "8763ca8707ec7874da874eb4c50c5d9214f7adff4f3f4893f91a99a6e9ce2210", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/002.json": "4a396ad196d1ead40eb7dbad06b78ccd9c94850f6d97ca258f1505e344d76320", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/004.json": "7dedaf9af777dd37adaeb4194066812f8a98f29193ba782411d7c1027e4331e2", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/011.json": "ee021afd67e356e95b8957d89927b0d99fdee12b0dd6085430cd1ef860bc758e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/030.json": "5d54d6d5a1d11a7ad3bedb6ec76dc444b60cf30b75d02d26eb13248f721a723e", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/022.json": "d81be7e968717804b83065b9025ba52b672b3352de5cb33ced59ac8ee9c4a745", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/016.json": "b6658ae25cae2cc31d0a2e5103714d0293db651daf9290fe691be263e2914e0f", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/010.json": "466a5166e6c1d394d5cd4bc8d63391453608f5ce635977e74478c11bf6b0e439", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/001.json": "e174d43f618cd998b5572d5a89dee05a809cc2270046c6ce4f2e1728387de1c8", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/021.json": "6b3740c952b2a21d8e47f7695c7fe0d05706a16a49e5457252371b54257e050b", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/009.json": "639322b6389d3408a53b1f22660fea442ca1c4b0be79191a8f45482e2c75521a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/012.json": "42424bbf2a4e692cd6eb632e04a0200069e4c8177a9e764b557a3c211c384a23", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/029.json": "70d87db7e576085281ea7641a25446fb1f5d4c9e6f110796e5e171661d98728a", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/history/026.json": "0dc5f21d899054928765a8a04fa48a090fb9eda47e1f2cb60a156991cd7b49e6", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/normalization/progress.json": "07a2190776d1826ed83dea62db64cd57d2f17f0a504e000a713f06d3c1b8c9ff", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/jobs/1/worker/normalization/standardization.json": "a6ca68193afaed449f42486d4fc1e42838a63ed4c3a9c2083771bb3154297aee", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/budget-stop-audit.json": "3bdaf94c4b4c211fc46ae156b08136aac07736e4bf13d42d8c4e9d20ce0d919f", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/budget-stop-git-freeze.json": "196d0dd5663ae508e49613dc339ed2371b1cf384d5995d0e0937ea22de3af235", + "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18/budget-stop-authorization.json": "d00807eeb0491eaa94e94f21c1748536694a58910c346b805c1d902d59f13591" + }, + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + }, + "comparison_contract": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", + "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "dispatch": "next_pending_fold_first_allowed_free_gpu; GPU2 reserved until Track A postfit terminal", + "no_retry": true, + "no_cross_card_retry": true, + "postfit_started": false, + "stop_after": "TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING", + "new_seed0_fits": 0, + "new_seed2_fits": 0, + "ensemble_started": false, + "dispatch_idle_fix_review": "ba4c62e8eec17d39258deab5759e8a849590cabbf4ace354288e609fcaeec1d1", + "accounting_approval": "1794990899.609759", + "semantic_approval": "1794990914.918719", + "original_seed1_root": "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18", + "previous_continuation_root": "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18", + "previous_stop_publication": "d3c94a00694c2e384efd3e002ca37d300866340e", + "continuation_number": 2, + "no_further_continuation": true, + "fresh_restart_folds": [ + 0, + 1 + ], + "previously_unattempted_folds": [ + 2, + 3, + 4 + ], + "old_attempts": [ + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18", + "gpu_seconds": 539.0720570000121 + }, + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18", + "gpu_seconds": 7943.241350897006 + } + ], + "old_seed1_gpu_once": 8482.313407897018, + "cumulative_seed1_gpu_cap": 167200, + "pending_rule": "median_current_started_projections_else_train_remaining_div5", + "active_unmeasured_rule": "max(actual_charge,initial_equal_share,current_measured_median); then included in started median", + "fold_stop_rule": "actual_charged_runaway_only", + "runaway_gpu_seconds": 40000, + "epoch_foreign_gpu_observation": "bounded observation-only snapshot after each completed epoch; errors retained, no causal or scheduling conclusion", + "accounting": "Runtime counters new-v2 only; cumulative seed1 adds old_attempts sum once; phase3 prior already contains both old attempts. Per-fold40000 is actual runaway, not partitioned reserve.", + "remaining_caps_final": true +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json new file mode 100644 index 000000000..562203003 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json @@ -0,0 +1,33 @@ +{ + "status": "PREPARED_PENDING_LOGIC_REVIEW", + "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "code_files": 51, + "source_files": 407, + "source_pins": "PASS", + "old_stop_freezes": [ + "a556b58340410dbe3faeb2dc2143b3708a7e724b", + "e3f251489d273037f47300ac295c174237551b7b" + ], + "old_attempts": [ + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18", + "gpu_seconds": 539.0720570000121 + }, + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18", + "gpu_seconds": 7943.241350897006 + } + ], + "old_seed1_gpu_once": 8482.313407897018, + "new_train_gpu_cap": 151517.68659210298, + "new_gpu_cap": 158717.68659210298, + "cumulative_seed1_gpu_cap": 167200, + "runaway_gpu_seconds": 40000, + "phase3_prior_gpu_once": 80903.62940728385, + "initial_pending_reserve": 30303.537318420596, + "metadata_cpu_seconds": 3.072146967, + "scientific_arrays_read": false, + "model_weights_read": false, + "test_targets_read": false, + "new_fits_launched": false +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py new file mode 100644 index 000000000..ef85535c9 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py @@ -0,0 +1,314 @@ +"""Metadata-only custody join, conservative components and deterministic S2 splits.""" +import argparse +import collections +import csv +import hashlib +import json +import math +from pathlib import Path +import platform +import shutil +import zipfile +import numpy as np + +BASE = Path('artifacts') +OUT = BASE / 's2-g1-components-2026-11-15' +REG = BASE / 's2-g1-registration-2026-11-14' +BAND = BASE / 's2-g1-band25-2026-11-15' +CMP = BASE / 's2-g1-band25-review-comparison-2026-11-15' +CONTRACT = Path('docs/s2-g1-component-split-contract-2026-11-15.md') +MERGE = 'MERGE_SAME_SIDE_LEAK_PROTECTION' +NOEDGE = 'NO_ADDITIONAL_EDGE_BOTH_UNRELATED' +EXPECTED = { + str(REG/'batch-complete.json'): 'b0ee7469c1ab755e809f8ff85ebc6ce940e9838a23eb091a4a7a0e91e77d0956', + str(REG/'contract.json'): 'c4271622e1775d430324dd8e8ad4524006184122079d2fc3bbb8fd0ba6a349d8', + str(REG/'custodian/plan.json'): '61f4695b7fdf16d747bdebafc1d0c6d362087e250e9d85a36ccce3b3caa71af6', + str(BAND/'custodian/mapping.json'): '7531566edf060cdae522b7cf32d0164d0cc6bd9d5b26be6a8d1c3a53bbf61142', + str(CMP/'comparison.csv'): 'e7603e06ac5529608c23b889475632b5d75cfbeea7686528460a6c81dcd10bfd', + str(CMP/'sources/mold/frozen-review.csv'): '6479eebdfc09629c7188f18f888494c5d9bea8dd160d92763813b15ef813ce18', + str(CMP/'sources/claude/band25-review.csv'): '93096734186c1247e46e98f26c7c9b0f7a4987c8d8d588e8147110ae3880be2e', +} + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for block in iter(lambda: f.read(1048576), b''): + h.update(block) + return h.hexdigest() + + +def read(path): + return json.loads(Path(path).read_text()) + + +def save(path, data): + with Path(path).open('x') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + f.write('\n') + + +class DSU: + def __init__(self, members): + self.parent = {x: x for x in members} + + def root(self, x): + while self.parent[x] != x: + self.parent[x] = self.parent[self.parent[x]] + x = self.parent[x] + return x + + def union(self, a, b): + a, b = sorted((self.root(a), self.root(b))) + self.parent[b] = a + + +def action(a, b): + allowed = {'unrelated', 'UNRESOLVED', 'rigid / same part'} + assert a in allowed and b in allowed + return NOEDGE if a == b == 'unrelated' else MERGE + + +def order(ids, stage, fold): + ids = sorted(ids) + rng = np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111409, stage, fold]))) + return [ids[int(i)] for i in rng.permutation(len(ids))] + + +def split_components(components): + by_id = {c['component_id']: c for c in components} + ids = sorted(by_id) + rank = {x: i for i, x in enumerate(order(ids, 0, 0))} + placement = sorted(ids, key=lambda x: (-by_id[x]['parts'], rank[x])) + folds = [[] for _ in range(5)] + parts = [0] * 5 + for x in placement: + k = min(range(5), key=lambda k: (parts[k], len(folds[k]), k)) + folds[k].append(x) + parts[k] += by_id[x]['parts'] + rows = [] + for k, test in enumerate(folds): + remaining = set(ids) - set(test) + val_order = order(remaining, 1, k) + val = val_order[:math.ceil(len(remaining)/5)] + learning = order(remaining-set(val), 2, k) + pools = {str(p): learning[:n] for p, n in [(25, math.ceil(len(learning)/4)), (50, math.ceil(len(learning)/2)), (100, len(learning))]} + reasons = [] + if len(ids) < 5: reasons.append('FEWER_THAN_FIVE_COMPONENTS') + if not test: reasons.append('EMPTY_TEST_FOLD') + if not val: reasons.append('EMPTY_VALIDATION_POOL') + if not learning: reasons.append('EMPTY_LEARNING_POOL') + if len({len(v) for v in pools.values()}) < 3: reasons.append('COINCIDENT_LEARNING_TIERS') + assert not(set(test)&set(val) or set(test)&set(learning) or set(val)&set(learning)) + assert set(test)|set(val)|set(learning) == set(ids) + assert pools['25'] == pools['50'][:len(pools['25'])] + assert pools['50'] == pools['100'][:len(pools['50'])] + rows.append(dict(fold=k, test=test, validation=val, validation_order=val_order, + learning_order=learning, pools=pools, noncomparability_reasons=reasons)) + assert collections.Counter(x for f in folds for x in f) == collections.Counter(ids) + return dict(order_version='g1-pcg64-seedsequence-v1', seed=2026111409, + numpy_version=np.__version__, placement_order=placement, folds=rows) + + +def components_stage(): + assert not OUT.exists(), 'Stage already exists: inspect rather than overwrite' + for p, digest in EXPECTED.items(): assert sha(p) == digest, p + for p, digest in read(CMP/'receipt.json')['source_sha256'].items(): + assert sha(CMP/p) == digest, p + for p, digest in read(REG/'contract.json')['input_sha256'].items(): assert sha(p) == digest, p + terminal = read(REG/'batch-complete.json') + assert terminal['status'] == 'ALL_REGISTRATIONS_COMPLETE' and terminal['failed_jobs'] == 0 + plan = read(REG/'custodian/plan.json') + assert len(terminal['jobs']) == len(plan['jobs']) == 2012 + assert {j['index'] for j in terminal['jobs']} == set(range(2012)) + assert all(j['returncode'] == 0 and j['worker']['status'] == 'PASS' and not j['worker']['retry'] for j in terminal['jobs']) + inventory_path = BASE/'s2-g1-inventory-2026-11-14/custodian/parts.json' + candidate_plan = BASE/'s2-g1-candidates-2026-11-14/custodian/plan.json' + old_join = BASE/'s2-g1-custody-handoff-2026-11-14/custodian/joined-frozen-review.json' + inventory = read(inventory_path) + by_case = {r['case']: r for r in inventory} + assert len(inventory) == len(by_case) == 624 + assert set(plan['geometry_cache']) == set(by_case) + dev = set(read(candidate_plan)['development_cases']) + old = read(old_join)['pairs'] + old_reviewed = {r[k] for r in old for k in ['a', 'b']} + assert all(by_case[r['a']]['source_key'] == by_case[r['b']]['source_key'] for r in old) + mapping = read(BAND/'custodian/mapping.json') + rows = list(csv.DictReader((CMP/'comparison.csv').open())) + original_fields = list(rows[0]) + assert len(rows) == 25 and len({r['anonymous_pair_id'] for r in rows}) == 25 + ids = {r['anonymous_pair_id'] for r in rows} + assert ids == {p['anonymous_pair_id'] for p in mapping['pairs']} + map_by_id = {p['anonymous_pair_id']: p for p in mapping['pairs']} + for r in rows: r['final_action'] = action(r['mold_relation'], r['claude_relation']) + assert {r['anonymous_pair_id'][:8] for r in rows if r['final_action'] == MERGE} == {'3058d0a6', '731fb76c', '1ead5efb', 'a7496efb'} + OUT.mkdir() + private = OUT/'custodian'; private.mkdir(mode=0o700) + public = OUT/'public'; public.mkdir() + shutil.copy2(CMP/'comparison.csv', public/'comparison-before-ruling.csv') + with (public/'comparison.csv').open('x') as f: + w = csv.DictWriter(f, original_fields+['final_action'], lineterminator='\n'); w.writeheader(); w.writerows(rows) + reread = list(csv.DictReader((public/'comparison.csv').open())) + assert [{k:r[k] for k in original_fields} for r in reread] == list(csv.DictReader((CMP/'comparison.csv').open())) + shutil.copy2(CONTRACT, public/'execution-contract.md') + input_sha = dict(EXPECTED) + for p in [inventory_path, candidate_plan, old_join, CONTRACT, Path(__file__)]: input_sha[str(p)] = sha(p) + source_groups = collections.defaultdict(list) + dsu = DSU(by_case) + edges = [] + for case, r in by_case.items(): source_groups[r['source_key']].append(case) + assert len(source_groups) == 249 + for key, members in sorted(source_groups.items()): + members.sort() + for b in members[1:]: + edges.append(dict(a=members[0], b=b, reason='SAME_SOURCE_KEY')) + dsu.union(members[0], b) + results = {} + counts = collections.Counter() + for i, pair in enumerate(plan['jobs']): + path = REG/f'custodian/results/pair-{i:04d}.json' + digest = sha(path); input_sha[str(path)] = digest + r = read(path) + assert r['index'] == i and r['pair'] == pair and r['simulation_fields_read'] is False + status = r['engineering_rule']['engineering_status']; counts[status] += 1 + results[i] = r + if status == 'LINK_UNDER_FIXED_RULE': + edges.append(dict(a=pair['a'], b=pair['b'], reason=status, registration_index=i, result_sha256=digest)) + dsu.union(pair['a'], pair['b']) + assert dict(counts) == {'OUTSIDE_FIXED_RULE':1897, 'BAND_REVIEW_REQUIRED':25, 'LINK_UNDER_FIXED_RULE':90} + assert {p['registration_index'] for p in mapping['pairs']} == {i for i,r in results.items() if r['engineering_rule']['engineering_status']=='BAND_REVIEW_REQUIRED'} + for path, digest in mapping['source_result_sha256'].items(): assert sha(path) == digest, path + joined = [] + band_reviewed = set() + for r in rows: + m = map_by_id[r['anonymous_pair_id']]; i = m['registration_index']; pair = plan['jobs'][i] + endpoints = {s['case'] for s in m['sides'].values()} + assert endpoints == {pair['a'], pair['b']} + band_reviewed.update(endpoints) + joined.append(dict(**r, registration_index=i, a=pair['a'], b=pair['b'], + engineering_rule=results[i]['engineering_rule'], result_sha256=input_sha[str(REG/f'custodian/results/pair-{i:04d}.json')])) + if r['final_action'] == MERGE: + edges.append(dict(a=pair['a'], b=pair['b'], reason=MERGE, anonymous_pair_id=r['anonymous_pair_id'], registration_index=i)) + dsu.union(pair['a'], pair['b']) + nodes = {}; node_sources = {}; exposure = {} + for case in sorted(by_case): + cache = plan['geometry_cache'][case]; path = Path(cache['path']) + assert sha(path) == cache['sha256'], case + with zipfile.ZipFile(path) as z: + info = z.getinfo('points.npy') + with z.open(info) as f: + version = np.lib.format.read_magic(f) + assert version == (1,0) + shape, fortran, dtype = np.lib.format.read_array_header_1_0(f) + header_bytes = f.tell() + assert len(shape) == 2 and shape[1] == 3 and shape[0] > 0 and not dtype.hasobject + assert info.file_size == header_bytes + math.prod(shape)*dtype.itemsize + nodes[case] = int(shape[0]) + node_sources[case] = dict(**cache, points_shape=shape, points_dtype=str(dtype), header_bytes=header_bytes, + points_crc32=info.CRC, points_uncompressed_bytes=info.file_size, payload_deserialized=False) + inv = by_case[case] + exposure[case] = dict(inventory_state=inv['exposure_state'], old_30_pair_endpoint=inv['old_30_pair_endpoint'], + individual_diagnostic_stages=inv['individual_diagnostic_stages'], calibration_development=case in dev, + blind120_geometry_review=case in old_reviewed, band25_geometry_review=case in band_reviewed, + all_corpus_historical_screening=inv['all_corpus_historical_screening'], unexposed_claim=False, + development_exposed=inv['exposure_state']=='KNOWN_DEVELOPMENT_EXPOSED' or case in dev) + groups = collections.defaultdict(list) + for case in sorted(by_case): groups[dsu.root(case)].append(case) + components = []; private_components = []; case_component = {} + for members in groups.values(): + cid = 'component-'+hashlib.sha256('\n'.join(members).encode()).hexdigest()[:16] + record = dict(component_id=cid, parts=len(members), nodes=sum(nodes[c] for c in members), + source_keys=len({by_case[c]['source_key'] for c in members}), + development_exposed_parts=sum(exposure[c]['development_exposed'] for c in members), + blind120_geometry_review_parts=sum(c in old_reviewed for c in members), + band25_geometry_review_parts=sum(c in band_reviewed for c in members), unexposed_claim=False) + components.append(record); private_components.append(dict(**record, members=members)) + case_component.update({c:cid for c in members}) + components.sort(key=lambda c:c['component_id']) + assert len({c['component_id'] for c in components}) == len(components) + assert all(case_component[e['a']] == case_component[e['b']] for e in edges) + key_public = []; key_private = [] + for i,(key, members) in enumerate(sorted(source_groups.items())): + kid = f'key-{i:03d}' + row = dict(key_id=kid, parts=len(members), nodes=sum(nodes[c] for c in members), component_id=case_component[members[0]]) + key_public.append(row); key_private.append(dict(**row, source_key=key, members=members)) + total_nodes = sum(nodes.values()) + largest = max(components, key=lambda c:(c['parts'], c['nodes'], c['component_id'])) + largest_nodes = max(components, key=lambda c:(c['nodes'], c['parts'], c['component_id'])) + def shares(c): return dict(**c, part_share=c['parts']/624, node_share=c['nodes']/total_nodes) + flags = [shares(c) for c in components if c['parts']*5 > 624 or c['nodes']*5 > total_nodes] + stats = dict(status='COMPONENTS_FROZEN_SPLITS_NOT_YET_RUN', parts=624, source_keys=249, nodes=total_nodes, + components=len(components), component_size_distribution=dict(sorted(collections.Counter(c['parts'] for c in components).items())), + source_key_size_distribution=dict(sorted(collections.Counter(len(m) for m in source_groups.values()).items())), + largest_by_parts=shares(largest), largest_by_nodes=shares(largest_nodes), oversized_report_flags=flags, + edge_counts=dict(collections.Counter(e['reason'] for e in edges)), registration_status_counts=dict(counts), + band_final_action_counts=dict(collections.Counter(r['final_action'] for r in rows)), + development_exposed_parts=sum(x['development_exposed'] for x in exposure.values()), + development_exposed_components=sum(c['development_exposed_parts']>0 for c in components), + labels_read=False, geometry_recomputed=False, equivalence_truth=False, cross_key_recall_certified=False, + ruling_message_ts='1794678062.264009', node_definition='Unique nodes used by tetrahedra; existing points.npy shape', + node_cache_sha256_all_verified=True, original_comparison_columns_preserved=True) + save(private/'joined-band-review.json', joined) + save(private/'edges.json', edges) + save(private/'components.json', private_components) + save(private/'source-keys.json', key_private) + save(private/'parts.json', [dict(case=c, source_key=by_case[c]['source_key'], component_id=case_component[c], nodes=nodes[c], exposure=exposure[c]) for c in sorted(by_case)]) + save(private/'node-metadata-provenance.json', node_sources) + save(private/'input-sha256.json', input_sha) + save(public/'components.json', components) + save(public/'parts-per-key.json', key_public) + save(public/'component-statistics.json', stats) + receipt = dict(status='COMPONENTS_FROZEN', ruling_message_ts='1794678062.264009', python=platform.python_version(), numpy=np.__version__, + source_sha256=EXPECTED, contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), + custody_artifact_sha256={p.name:sha(p) for p in sorted(private.iterdir())}, + public_artifact_sha256={p.name:sha(p) for p in sorted(public.iterdir())}, + reviewer_mapping_returned=False, original_source_files_unchanged=all(sha(p)==h for p,h in EXPECTED.items())) + save(public/'component-receipt.json', receipt) + for p in private.iterdir(): p.chmod(0o600) + print(json.dumps(stats, ensure_ascii=False)) + + +def split_stage(report_ts): + public = OUT/'public'; private = OUT/'custodian' + assert not (public/'splits.json').exists(), 'Splits already exist' + stats = read(public/'component-statistics.json') + assert not stats['oversized_report_flags'] or report_ts, 'Report oversized components before splitting' + receipt = read(public/'component-receipt.json') + assert sha(CONTRACT) == receipt['contract_sha256'] and sha(__file__) == receipt['code_sha256'] + for name, digest in receipt['custody_artifact_sha256'].items(): assert sha(private/name) == digest + for name, digest in receipt['public_artifact_sha256'].items(): assert sha(public/name) == digest + components = read(public/'components.json'); by_id = {c['component_id']:c for c in components} + splits = split_components(components) + def summarize(ids): + return dict(components=len(ids), parts=sum(by_id[c]['parts'] for c in ids), nodes=sum(by_id[c]['nodes'] for c in ids), + development_exposed_parts=sum(by_id[c]['development_exposed_parts'] for c in ids), + development_exposed_components=sum(by_id[c]['development_exposed_parts']>0 for c in ids), unexposed_claim=False) + for fold in splits['folds']: + fold['counts'] = dict(test=summarize(fold['test']), validation=summarize(fold['validation']), + pools={p:summarize(ids) for p,ids in fold['pools'].items()}) + save(public/'splits.json', splits) + private_rows = [] + for part in read(private/'parts.json'): + cid = part['component_id']; assignments = [] + for f in splits['folds']: + role = 'test' if cid in f['test'] else 'validation' if cid in f['validation'] else 'learning' + assignments.append(dict(fold=f['fold'], role=role, learning_pools=[p for p,ids in f['pools'].items() if cid in ids])) + private_rows.append(dict(**part, assignments=assignments)) + save(private/'part-splits.json', private_rows) + (private/'part-splits.json').chmod(0o600) + result = dict(status='COMPONENTS_AND_FIVE_FOLD_NESTED_POOLS_FROZEN_AWAITING_REVIEW', + contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), components_receipt_sha256=sha(public/'component-receipt.json'), + splits_sha256=sha(public/'splits.json'), custody_part_splits_sha256=sha(private/'part-splits.json'), + oversized_components_report_message_ts=report_ts, no_component_cut=True, + all_required_edges_same_fold=True, component_test_partition_verified=True, nested_pools_verified=True, + no_geometry_registration_review_rerun=True, training_pilot_gpu=False, + fold_counts=[dict(fold=f['fold'], **f['counts'], noncomparability_reasons=f['noncomparability_reasons']) for f in splits['folds']]) + save(public/'split-receipt.json', result) + print(json.dumps(result, ensure_ascii=False)) + + +if __name__ == '__main__': + p = argparse.ArgumentParser(); p.add_argument('stage', choices=['components','splits']); p.add_argument('--oversized-report-ts') + args = p.parse_args() + if args.stage == 'components': components_stage() + else: split_stage(args.oversized_report_ts) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py new file mode 100644 index 000000000..723c9a487 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py @@ -0,0 +1,42 @@ +"""Fixed Gre24 model choices; no test-driven selection or node subsampling.""" +import numpy as np + + +def xgb_model(seed,threads=8): + from xgboost import XGBRegressor + return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1, + objective='reg:squarederror',random_state=seed,n_jobs=threads) + + +def fnn_model(): + import torch + layers=[];width=19 + for _ in range(4): + layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100 + layers.append(torch.nn.Linear(100,1)) + return torch.nn.Sequential(*layers) + + +def eligible_roles(split,successful): + successful=set(successful) + roles={role:[c for c in split[role] if c in successful] + for role in ['train','test','fnn_learning','fnn_validation']} + if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap') + if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap') + if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']): + raise ValueError('FNN learning/validation do not partition train') + missing={role:[c for c in split[role] if c not in successful] for role in roles} + return roles,missing + + +def require_training_role(case,split,role): + if role not in ['train','fnn_learning','fnn_validation']: + raise PermissionError('Only frozen training-side roles may read targets') + if case in split['test'] or case not in split[role]: + raise PermissionError('Case is not in the requested training role') + + +def early_stop_update(best,stale,score,patience=5): + if not np.isfinite(score):raise ValueError('Nonfinite validation MSE') + improved=score=patience) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py new file mode 100644 index 000000000..4e3495ed7 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py @@ -0,0 +1,34 @@ +"""Single-writer Git barriers; worker verification never spawns a process.""" +import fcntl +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +from phase2_h1_worker_v2 import sha,checked,save +BASE=Path('/home/frank14f/optane/MeltMind') +WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') + + +def freeze(root,name,files): + with Path('/tmp/meltmind-single-writer.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip() + if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean') + hashes={} + for path in files: + path=Path(path);rel=str(path.resolve().relative_to(BASE));dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(path,dest);hashes[rel]=sha(path) + subprocess.run(['git','-C',str(WRITER),'add','--',*hashes],check=True) + if git('diff','--cached','--name-only'):subprocess.run(['git','-C',str(WRITER),'commit','-m','Phase2 '+root.name+' '+name+' immutable barrier'],check=True) + commit=git('rev-parse','HEAD') + for rel,h in hashes.items(): + if hashlib.sha256(subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+rel])).hexdigest()!=h:raise ValueError('Git byte mismatch') + save(root/(name+'-git-freeze.json'),dict(commit=commit,sha256=hashes,remote_published=False)) + save(root/(name+'-authorization.json'),dict(proof_sha256=sha(root/(name+'-git-freeze.json')),contract_sha256=sha(root/'contract.json'))) + + +def verify(root,name): + auth=json.loads((root/(name+'-authorization.json')).read_text());checked(root/'contract.json',auth['contract_sha256']) + p=checked(root/(name+'-git-freeze.json'),auth['proof_sha256']);proof=json.loads(p.read_text()) + for rel,h in proof['sha256'].items():checked(BASE/rel,h) + return proof diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py new file mode 100644 index 000000000..1524f0cd4 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py @@ -0,0 +1,20 @@ +"""Exact semantic PASS and immutable source/contract identity before custody.""" +import json +from pathlib import Path +from phase2_h1_worker_v2 import sha,checked +from phase2_home_freeze_io import verify + +def read(p):return json.loads(Path(p).read_text()) +def validate_review(root): + root=Path(root);c=read(root/'contract.json');proof=verify(root,'implementation');binding=read(root/'expected-review-binding.json');review=read(root/'review.json') + expected=dict(reviewed_commit=proof['commit'],contract_sha256=sha(root/'contract.json'),implementation_proof_sha256=sha(root/'implementation-git-freeze.json'),comparison_contract_sha256=c['comparison_contract_sha256'],code_sha256=c['code_sha256']) + if binding!=expected:raise PermissionError('Review binding differs from implementation') + if review.get('verdict')!='PASS':raise PermissionError('Actual exact PASS required') + for name,value in expected.items(): + if review.get(name)!=value:raise PermissionError('Review identity mismatch: '+name) + checked(c['comparison_contract'],expected['comparison_contract_sha256']) + for p,h in expected['code_sha256'].items():checked(p,h) + frozen=verify(root,'implementation-review') + for p in [root/'review.json',root/'expected-review-binding.json',root/'contract.json']: + if frozen['files'].get(str(p.resolve()))!=sha(p):raise PermissionError('Review freeze does not include exact required file') + return review diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py new file mode 100644 index 000000000..e9d4b0069 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py @@ -0,0 +1,179 @@ +"""H1-RES fixed-XGB worker. No queue or implicit execution approval. + +The reviewed contract binds training/test roles, source metadata, precision, +clock weighting and source hashes. Test predictions never load target arrays. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time +import traceback + +import numpy as np +from threadpoolctl import threadpool_limits +from gre24_models import xgb_model +from phase2_residual_math import learning_clock, residual_target, decode_seconds + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as source: + for block in iter(lambda: source.read(4*1024**2), b''): h.update(block) + return h.hexdigest() + + +def save(path, value): + path = Path(path); tmp = path.with_suffix(path.suffix+'.tmp') + tmp.write_text(json.dumps(value, indent=2, allow_nan=False)+'\n'); tmp.replace(path) + + +def checked(path, digest): + if sha(path) != digest: raise ValueError('Pinned file mismatch: '+str(path)) + return Path(path) + + +def contract(root): + import xgboost + if np.__version__ != '1.26.4' or xgboost.__version__ != '2.1.4': + raise ValueError('Frozen library version differs') + c = json.loads((root/'contract.json').read_text()) + if c.get('version') != 'phase2-h1-res-v1' or not c.get('main_review'): + raise PermissionError('No fixed reviewed H1 contract') + for p, h in c['code_sha256'].items(): checked(p, h) + for p, h in c['source_sha256'].items(): checked(p, h) + return c + + +def role_rows(c, job, role): + if role not in ('learning', 'test'): raise ValueError('Unknown role') + if set(job['learning']) & set(job['test']): raise PermissionError('Training/test overlap') + if len(job[role]) != len(set(job[role])): raise ValueError('Repeated case') + # Contract builder must cross-check these frozen roles against original + # source contracts. Here a role cannot be expanded through row metadata. + return [c['parts'][job['protocol']][case] for case in job[role]] + + +def load_features(row): + source = Path(row['output']); hashes = row['output_sha256'] + f = np.load(checked(source/'Vf.npy', hashes['Vf.npy']), allow_pickle=False) + x = np.load(checked(source/'features.npy', hashes['features.npy']), mmap_mode='r', allow_pickle=False) + ids = np.load(checked(source/'raw_ids.npy', hashes['raw_ids.npy']), allow_pickle=False) + if x.shape != (row['nodes'], 19) or f.shape != (row['nodes'],) or ids.shape != f.shape: + raise ValueError('Feature/node identity shape mismatch') + if f.dtype != np.dtype('float64') or not np.array_equal(np.asarray(x[:, 0],dtype=np.float32), f.astype(np.float32)): + raise ValueError('Original float64 F differs from frozen first column') + if np.any(np.diff(ids) <= 0) or not np.isfinite(x).all(): raise ValueError('Bad IDs/features') + return np.asarray(x, dtype=np.float32), f, ids + + +def fit(root, c, job): + out = root/'fits'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False) + save(out/'started.json', {'unix': time.time(), 'job': job, 'contract_sha256': sha(root/'contract.json')}) + cpu = time.process_time(); access = []; rows = role_rows(c, job, 'learning') + end_ratios, groups = [], [] + # First pass: only learning-role retained-tet targets for this cell's clock. + for row in rows: + path = Path(row['output'])/'seconds.npy' + access.append({'case': row['case'], 'role': 'learning', 'purpose': 'clock', 'path': str(path)}) + save(out/'target-access.json', access) + y = np.load(checked(path, row['output_sha256']['seconds.npy']), mmap_mode='r', allow_pickle=False) + if y.shape != (row['nodes'],) or not np.isfinite(y).all() or np.any(y < 0): raise ValueError('Invalid learning target') + if y.dtype != np.dtype('float64') or not np.isfinite(row['t_set']) or row['t_set']<=0: raise ValueError('Endpoint precision/scale mismatch') + raw=np.load(checked(Path(row['output'])/'raw_ids.npy',row['output_sha256']['raw_ids.npy']),allow_pickle=False) + if raw.shape!=y.shape or np.any(np.diff(raw)<=0): raise ValueError('Endpoint raw ID mismatch') + end_ratios.append(float(y.max())/float(row['t_set'])); groups.append(row['clock_group']) + clock = learning_clock(end_ratios, groups) + save(out/'clock.json', {'c': clock, 'source': 'learning retained-tet max(T)/t_set, mean within component then mean across components', + 'parts': [row['case'] for row in rows], 'end_ratios': end_ratios, 'groups': groups}) + n = sum(row['nodes'] for row in rows); x = np.empty((n, 19), np.float32); y = np.empty(n, np.float32); cursor = 0 + for row in rows: + xx, f, ids = load_features(row) + path = checked(Path(row['output'])/'target.npy', row['output_sha256']['target.npy']) + target = np.load(path, mmap_mode='r', allow_pickle=False) + physical=np.load(checked(Path(row['output'])/'seconds.npy',row['output_sha256']['seconds.npy']),mmap_mode='r',allow_pickle=False) + if not np.array_equal(target,physical/row['t_set']): raise ValueError('Frozen normalized target differs from physical seconds/t_set') + count = len(f); x[cursor:cursor+count] = xx + y[cursor:cursor+count] = residual_target(target, f, clock, fit_dtype='float32'); cursor += count + access.append({'case': row['case'], 'role': 'learning', 'purpose': 'residual_fit', 'path': str(path)}) + save(out/'target-access.json', access) + with threadpool_limits(job['threads']): + model = xgb_model(job['seed'], job['threads']) + save(out/'progress.json', {'stage': 'FIT', 'nodes': n, 'clock': clock}) + model.fit(x, y, verbose=False) + checkpoint = out/'model.ubj'; model.save_model(checkpoint) + (out/'xgboost-config.json').write_text(model.get_booster().save_config()) + save(out/'result.json', {'status': 'FIT_COMPLETE_NO_TEST_SCORES', 'checkpoint': str(checkpoint), + 'checkpoint_sha256': sha(checkpoint), 'clock_sha256': sha(out/'clock.json'), 'c': clock, + 'learning_nodes': n, 'learning_parts': len(rows), 'cpu_seconds': time.process_time()-cpu, + 'test_labels_read': False, 'new_decodes': 0, 'retry': False}) + + +def predict(root, c, job): + import joblib + import xgboost + proof = json.loads((root/'models-git-freeze.json').read_text()) + if not proof.get('commit') or not proof.get('checkout'): + raise PermissionError('All-terminal model Git freeze missing or mismatched') + for rel in ['contract.json', 'batch-complete.json', 'models-terminal.json']: + path = root/rel; expected = proof['sha256'][rel]; checked(path, expected) + repository_path = str(path.resolve().relative_to(Path.cwd())) + raw = subprocess.check_output(['git', 'show', proof['commit']+':'+repository_path], cwd=proof['checkout']) + if hashlib.sha256(raw).hexdigest() != expected: raise PermissionError('Git freeze differs from local files') + terminal = json.loads((root/'batch-complete.json').read_text()) + if terminal['global_stop'] or not terminal['accounting_complete'] or terminal['attempted'] != len(c['jobs']): + raise PermissionError('Fit queue not complete') + freeze = json.loads((root/'models-terminal.json').read_text()) + if len(freeze['models']) != len(c['jobs']): raise PermissionError('Incomplete model manifest') + entry = next(r for r in freeze['models'] if r['index'] == job['index']) + if entry['status'] != 'PASS': raise ValueError('No checkpoint for failed cell') + out = root/'predictions'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False) + model = xgboost.XGBRegressor(); model.load_model(checked(entry['checkpoint'], entry['checkpoint_sha256'])) + old = job['baseline_model']; checked(old['path'], old['sha256']) + if old['format'] == 'joblib': + baseline = joblib.load(old['path']) + elif old['format'] == 'ubj': + baseline = xgboost.XGBRegressor(); baseline.load_model(old['path']) + else: + raise ValueError('Unsupported frozen A checkpoint format') + clock = json.loads(checked(entry['clock_path'], entry['clock_sha256']).read_text())['c'] + index = []; cpu = time.process_time() + with threadpool_limits(job['threads']): + for row in role_rows(c, job, 'test'): + x, f, ids = load_features(row) # No target.npy load on this path. + seconds = decode_seconds(model.predict(x), f, clock, row['t_set']) + old_seconds = np.asarray(baseline.predict(x), dtype=np.float64)*row['t_set'] + if old_seconds.shape != seconds.shape or not np.isfinite(old_seconds).all(): raise ValueError('Invalid A prediction') + if job['protocol']=='grouped': + prior=c['baseline_predictions'][str(job['index'])][row['case']] + with np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as z: + iq=np.searchsorted(ids,z['raw_ids']) + if np.any(iq>=len(ids)) or not np.array_equal(ids[iq],z['raw_ids']):raise ValueError('Historical A query identity differs') + if not np.array_equal(old_seconds[iq],z[prior['array']]):raise ValueError('Historical A query prediction differs') + path = out/(row['case']+'.npz') + np.savez_compressed(path, seconds=seconds, A_seconds=old_seconds, cF_seconds=clock*f*row['t_set'], + EIK_seconds=f*row['t_set'], raw_ids=ids) + index.append({'case': row['case'], 'path': str(path), 'sha256': sha(path), + 'negative_predictions': int(np.sum(seconds < 0)), 'nodes': len(ids)}) + save(out/'progress.json', {'parts': len(index), 'cpu_seconds': time.process_time()-cpu}) + save(out/'index.json', index) + save(out/'result.json', {'status': 'PREDICTIONS_COMPLETE_NO_SCORES', 'index_sha256': sha(out/'index.json'), + 'cpu_seconds': time.process_time()-cpu, 'target_values_read': False}) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path) + parser.add_argument('mode', choices=['fit', 'predict']); parser.add_argument('index', type=int) + args = parser.parse_args(); start = time.process_time() + try: + config = contract(args.root); selected = config['jobs'][args.index] + if selected['index'] != args.index: raise ValueError('Job index mismatch') + (fit if args.mode == 'fit' else predict)(args.root, config, selected) + except Exception as exc: + dest = args.root/'failures'; dest.mkdir(exist_ok=True) + save(dest/f'{args.mode}-{args.index:02d}.json', {'type': type(exc).__name__, 'message': str(exc), + 'traceback': traceback.format_exc(), 'pid': os.getpid(), 'cpu_seconds': time.process_time()-start, + 'retry': False}) + raise diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py new file mode 100644 index 000000000..cd9badde7 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py @@ -0,0 +1,87 @@ +"""Frozen H1 interface and label-scoped H2 inputs; no test target access.""" +import json +from pathlib import Path +import time +import numpy as np +from scipy.sparse import csr_matrix +from scipy.sparse.csgraph import connected_components +import torch +from phase2_h1_worker_v2 import checked,sha,save +from phase2_h2_model import edge_attributes +from phase2_residual_math import residual_target + +BASE=Path('/home/frank14f/optane/MeltMind') + +def read(p):return json.loads(Path(p).read_text()) + +def config(root): + c=read(root/'contract.json') + if c['approval']!='1794900702.867059' or c['mode']!='PILOT_ONLY':raise PermissionError('No approved pilot contract') + for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h) + if torch.__version__!=c['versions']['torch'] or np.__version__!=c['versions']['numpy']:raise ValueError('Library identity changed') + return c + +def cached(c,case,name): + r=c['parts'][case];return np.load(checked(Path(r['output'])/(name+'.npy'),r['output_sha256'][name+'.npy']),allow_pickle=False) + +def standardization(c,root): + n=0;mean=np.zeros(19,np.float64);M2=np.zeros(19,np.float64);seen=[];start=time.process_time() + for case in c['folds'][0]['learning']: + x=np.asarray(cached(c,case,'features'),np.float64) + if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid') + k=len(x);avg=x.mean(0);delta=avg-mean;batch=np.square(x-avg).sum(0) + M2+=batch+delta*delta*(n*k/(n+k));mean+=delta*(k/(n+k));n+=k;seen.append(case) + save(root/'progress.json',dict(stage='LEARNING_ONLY_STANDARDIZATION',parts=len(seen),nodes=n,cpu_seconds=time.process_time()-start,labels_read=False)) + std=np.sqrt(M2/n);scale=np.where(std==0,1.,std) + if not np.isfinite(scale).all() or n!=c['folds'][0]['learning_nodes']:raise ValueError('Standardization coverage') + out=dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),nodes=n,parts=seen,labels_read=False,cpu_seconds=time.process_time()-start) + save(root/'standardization.json',out);return out + +def load_part(c,case,stats,role,root): + allowed=c['folds'][0][role] if role in ('learning','validation') else [] + if case not in allowed:raise PermissionError('Pilot role violation') + r=c['parts'][case];g=c['geometry'][case] + with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z: + points=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy() + indptr=z['indptr'].copy();sender=z['indices'].astype(np.int64);old_rg=float(z['rg']) + n=len(raw);receiver=np.repeat(np.arange(n,dtype=np.int64),np.diff(indptr)) + if not np.array_equal(raw,cached(c,case,'raw_ids')) or not np.array_equal(mass,cached(c,case,'mass')):raise ValueError('Frozen geometry/node mass identity') + if n!=r['nodes'] or len(sender)!=g['directed_edges']:raise ValueError('Frozen graph counts') + a,rg=edge_attributes(points,mass,receiver,sender) + if not np.isclose(rg,old_rg,rtol=1e-10,atol=1e-12):raise ValueError('Original mass Rg differs') + graph=csr_matrix((np.ones(len(sender)),sender,indptr),shape=(n,n));count,comp=connected_components(graph,directed=False) + x=cached(c,case,'features');F=cached(c,case,'Vf') + if F.dtype!=np.float64 or F.shape!=(n,) or not np.isfinite(F).all():raise ValueError('Original F must be FP64') + x=((np.asarray(x,np.float64)-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32) + if not np.isfinite(x).all():raise ValueError('Standardized feature overflow') + result=dict(x=torch.from_numpy(x),mass=torch.from_numpy(mass.astype(np.float32)),receiver=torch.from_numpy(receiver),sender=torch.from_numpy(sender),attributes=torch.from_numpy(a),component_ids=torch.from_numpy(comp.astype(np.int64)), + F=F,raw_ids=raw,physical_components=int(count),nodes=n,directed_edges=len(sender),Rg=rg,t_set=r['t_set']) + if role=='learning': + # Append the intended role/hash before opening any cached target. + with (root/'learning-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,role=role,target_sha256=r['output_sha256']['target.npy'],clock_sha256=c['folds'][0]['clock_sha256']))+'\n') + y=cached(c,case,'target');result['target']=torch.from_numpy(residual_target(y,F,c['folds'][0]['c'],fit_dtype='float32')) + return result + +def to_device(data,device): + return {k:(v.to(device) if isinstance(v,torch.Tensor) else v) for k,v in data.items()} + +def arguments(data):return tuple(data[k] for k in ['x','mass','receiver','sender','attributes','component_ids']) + +def projection(c,epoch_seconds,validation_seconds,pilot_seconds): + f0=c['folds'][0];per_fold=[] + for f in c['folds']: + learning_ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],len(f['learning'])/len(f0['learning'])) + validation_ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation'])) + prediction_ratio=max(f['test_nodes']/f0['validation_nodes'],f['test_edges']/f0['validation_edges'],len(f['test'])/len(f0['validation'])) + per_fold.append(dict(fold=f['fold'],train_200_seconds=200*epoch_seconds*learning_ratio,validation_20_seconds=20*validation_seconds*validation_ratio,test_prediction_seconds=validation_seconds*prediction_ratio, + learning_ratio=learning_ratio,validation_ratio=validation_ratio,test_ratio=prediction_ratio)) + # Every measured forward/backward time includes load, transfers and optimizer. + training=1.25*sum(f['train_200_seconds']+f['validation_20_seconds'] for f in per_fold) + prediction=1.25*sum(f['test_prediction_seconds'] for f in per_fold) + total=pilot_seconds+training+prediction+c['future_setup_reserve_seconds'] + passed=training<=12*3600 and prediction<=2*3600 and total<=16*3600 + return dict(per_fold=per_fold,measured_full_epoch_seconds=epoch_seconds,measured_validation_inference_seconds=validation_seconds, + safety_factor=1.25,training_with_validation_seconds=training,prediction_seconds=prediction,pilot_seconds=pilot_seconds,future_setup_seconds=c['future_setup_reserve_seconds'],total_gpu_seconds=total, + fits_12h_training_2h_prediction_16h_total=passed,decision='COST_FEASIBLE_FULL_SCOPE_AUTHORIZED' if passed else 'PARKED_RESOURCE', + full_run_authorized_by='1794900702.867059',full_run_started=False,epoch_target=200,folds=5,new_fits=5,validation_targets_read=False, + limitation='Validation forward timing only; no target metrics in pilot. CPU final field/topology scoring is unmeasured and cannot be claimed affordable from this GPU projection.') diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py new file mode 100644 index 000000000..cf9a65b51 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py @@ -0,0 +1,108 @@ +"""H2-GEO reviewed fixed FP32 graph/token recipe; no data or launches.""" +import math +import numpy as np +import torch +from torch import nn +from torch.utils.checkpoint import checkpoint + +EDGE_CHUNK=65536 +NODE_CHUNK=8192 +ATOL=3e-5 +RTOL=3e-4 + +def lr_epoch(epoch): + if not 1<=epoch<=200:raise ValueError('Fixed 1-based epochs 1..200') + if epoch<=10:return 1e-4+(epoch-1)*9e-4/9 + return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(epoch-10)/190))/2 + +def namespace_components(part_ids,local_ids): + keys=list(zip(part_ids,local_ids));lookup={k:i for i,k in enumerate(sorted(set(keys)))} + return np.array([lookup[k] for k in keys],np.int64) + +def edge_attributes(points,mass,receiver,sender): + x=np.asarray(points,np.float64);m=np.asarray(mass,np.float64) + if not np.isfinite(x).all() or not np.isfinite(m).all() or np.any(m<=0):raise ValueError('Invalid geometry/mass') + center=np.sum(x*m[:,None],axis=0)/m.sum() + rg=float(np.sqrt(np.sum(m*np.sum((x-center)**2,axis=1))/m.sum())) + if not math.isfinite(rg) or rg<=0:raise ValueError('Invalid mass Rg') + xi=(x-center)/rg;delta=xi[sender]-xi[receiver];length=np.linalg.norm(delta,axis=1) + if np.any(length<=0):raise ValueError('Nonpositive true edge') + return np.column_stack([delta,length]).astype(np.float32),rg + +class GeoBlock(nn.Module): + def __init__(self): + super().__init__() + self.local_norm=nn.LayerNorm(64);self.global_norm=nn.LayerNorm(64);self.ffn_norm=nn.LayerNorm(64) + self.edge=nn.Sequential(nn.Linear(4,16),nn.GELU(),nn.Linear(16,1),nn.Softplus()) + self.local_value=nn.Linear(64,64,bias=False) + self.assignment=nn.Linear(64,64);self.global_value=nn.Linear(64,64,bias=False) + self.attention=nn.MultiheadAttention(64,4,dropout=0,batch_first=True) + self.ffn=nn.Sequential(nn.Linear(64,128),nn.GELU(),nn.Linear(128,64)) + + def local(self,h,m,receiver,sender,attributes,edge_chunk): + values=self.local_value(h);num=h.new_zeros(h.shape);den=h.new_zeros((len(h),1)) + for lo in range(0,len(sender),edge_chunk): + hi=lo+edge_chunk;r=receiver[lo:hi];s=sender[lo:hi] + weight=self.edge(attributes[lo:hi])*m[s,None] + num.index_add_(0,r,weight*values[s]);den.index_add_(0,r,weight) + occupied=torch.bincount(receiver,minlength=len(h))>0 + if not bool(torch.isfinite(den).all()) or bool((den[occupied]<=0).any()):raise ValueError('Invalid local denominator') + safe=torch.where(occupied[:,None],den,torch.ones_like(den)) + return num/safe + + def global_message(self,h,m,components,node_chunk,detach_global=False): + out=h.new_zeros(h.shape) + for ids in components: + numerator=h.new_zeros((64,64));denominator=h.new_zeros((64,1));chunks=[] + for lo in range(0,len(ids),node_chunk): + ix=ids[lo:lo+node_chunk];hh=h[ix] + assignment=torch.softmax(self.assignment(hh),dim=-1) + weighted=assignment*m[ix,None] + numerator=numerator+weighted.T@self.global_value(hh) + denominator=denominator+weighted.sum(0)[:,None] + chunks.append((ix,assignment)) + if not bool(torch.isfinite(denominator).all()) or bool((denominator<=0).any()):raise ValueError('Invalid token denominator') + z=numerator/denominator + if detach_global:z=z.detach() # Only the synthetic negative control enables this. + z=self.attention(z[None],z[None],z[None],need_weights=False)[0][0] + for ix,assignment in chunks:out.index_copy_(0,ix,assignment@z) + return out + + def forward(self,h,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global=False): + h=h+self.local(self.local_norm(h),m,receiver,sender,attributes,edge_chunk) + h=h+self.global_message(self.global_norm(h),m,components,node_chunk,detach_global) + return h+self.ffn(self.ffn_norm(h)) + +class H2Geo(nn.Module): + def __init__(self): + super().__init__();self.encoder=nn.Sequential(nn.Linear(19,64),nn.GELU(),nn.Linear(64,64)) + self.blocks=nn.ModuleList([GeoBlock() for _ in range(4)]) + self.head=nn.Sequential(nn.Linear(83,64),nn.GELU(),nn.Linear(64,1)) + nn.init.zeros_(self.head[-1].weight);nn.init.zeros_(self.head[-1].bias) + + def forward(self,x,m,receiver,sender,attributes,component_ids,*,edge_chunk=EDGE_CHUNK,node_chunk=NODE_CHUNK,checkpoint_blocks=True,detach_global=False): + if x.dtype!=torch.float32 or m.dtype!=torch.float32 or attributes.dtype!=torch.float32:raise ValueError('Only FP32 model/aggregations') + if not bool(torch.isfinite(m).all()) or bool((m<=0).any()):raise ValueError('Nonpositive mass; no epsilon') + if bool((component_ids[receiver]!=component_ids[sender]).any()):raise ValueError('Cross-component edge') + components=[torch.nonzero(component_ids==c,as_tuple=True)[0] for c in torch.unique(component_ids)] + h=self.encoder(x) + for block in self.blocks: + def run(v,b=block):return b(v,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global) + h=checkpoint(run,h,use_reentrant=False,preserve_rng_state=False) if checkpoint_blocks and self.training and torch.is_grad_enabled() else run(h) + outputs=[] + for lo in range(0,len(x),node_chunk):outputs.append(self.head(torch.cat([h[lo:lo+node_chunk],x[lo:lo+node_chunk]],dim=-1))[:,0]) + return torch.cat(outputs) + +def optimizer(model): + decay=[];no_decay=[] + for name,p in model.named_parameters(): + (no_decay if name.endswith('bias') or p.ndim==1 else decay).append(p) + return torch.optim.AdamW([dict(params=decay,weight_decay=1e-4),dict(params=no_decay,weight_decay=0.)],lr=lr_epoch(1),betas=(.9,.999),eps=1e-8) + +def weighted_loss(prediction,target,parts,part_nodes,total_nodes): + if len(prediction)!=part_nodes or prediction.shape!=target.shape:raise ValueError('Full part loss shape') + return torch.mean((prediction-target)**2)*(parts*part_nodes/total_nodes) + +def comparison(a,b): + a=a.detach();b=b.detach();difference=float((a-b).abs().max());scale=float(a.abs().max()) + return dict(max_absolute=difference,reference_scale=scale,relative_to_scale=difference/max(scale,1e-30),passed=difference<=ATOL+RTOL*scale) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py new file mode 100644 index 000000000..32250c36c --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py @@ -0,0 +1,80 @@ +"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing.""" +import gc +import json +import os +from pathlib import Path +import random +import sys +import time +import traceback +import numpy as np +import torch +from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection +from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch +from phase2_h1_worker_v2 import save,sha +from phase2_residual_math import decode_seconds + +def peaks(): + return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved())) + +def seed(): + random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0) + +def train_step(model,opt,data,parts,total): + model.train();opt.zero_grad(set_to_none=True) + prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total) + if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss') + loss.backward() + norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True) + opt.step() + if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter') + torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks()) + +def run(root): + c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0 + (out/'history').mkdir() + def record(value): + nonlocal history_index + item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'history'/f'{history_index:05d}.json',item);history_index+=1 + save(out/'progress.json',item);return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs') + stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() + props=torch.cuda.get_device_properties(0) + save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0)))) + fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD' + for case in c['representatives']: + seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize() + for step in range(2): + tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r)) + records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks())) + del data,opt,model;gc.collect();torch.cuda.empty_cache() + save(out/'representative-timings.json',records) + stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[] + for index in order: + case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total) + trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data + torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start + if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch') + for group in opt.param_groups:group['lr']=lr_epoch(2) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt') + save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt'))) + stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize() + validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds + duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start + projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected) + save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise + +if __name__=='__main__':run(Path(sys.argv[1])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py new file mode 100644 index 000000000..c1d5744bc --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py @@ -0,0 +1,63 @@ +"""Metadata-only pilot contract from frozen H1 inputs; never accesses labels.""" +import ast +import json +from pathlib import Path +import sys +import numpy as np +import torch +from phase2_h1_worker_v2 import sha,save,checked + +BASE=Path('/home/frank14f/optane/MeltMind') + +def closure(names): + pending=list(names);done={} + while pending: + name=pending.pop();path=BASE/'tools'/Path(*name.split('.')).with_suffix('.py') + if not path.exists() or name in done:continue + done[name]=path + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node,ast.Import):pending.extend(a.name for a in node.names) + elif isinstance(node,ast.ImportFrom) and node.module:pending.append(node.module) + return {str(p.relative_to(BASE)):sha(p) for p in done.values()} + +def run(acceptance): + acc=json.loads(acceptance.read_text()) + if acc['status']!='PASS':raise ValueError('Final acceptance not PASS') + for p,h in acc['code_sha256'].items():checked(BASE/p,h) + oldroot=BASE/'artifacts/phase2-h1-execution-2026-11-17';sources={} + def read(p): + p=Path(p);sources[str(p.relative_to(BASE))]=sha(p);return json.loads(p.read_text()) + old=read(oldroot/'contract.json');models=read(oldroot/'models-terminal.json')['models'];gm=read(oldroot/'geometry/manifest.json') + geometry={r['case']:r for r in gm['records'] if r['protocol']=='grouped'};folds=[] + for fold in range(5): + job=next(j for j in old['jobs'] if j['protocol']=='grouped' and j['pool']==100 and j['fold']==fold) + model=next(m for m in models if m['index']==job['index']);assert model['status']=='PASS';checked(model['clock_path'],model['clock_sha256']);clock=read(model['clock_path']);assert clock['parts']==job['learning'] + rows=old['grouped_rows'];validation=sorted(case for case,r in rows.items() if next(a for a in r['assignments'] if a['fold']==fold)['role']=='validation') + f=dict(fold=fold,learning=job['learning'],validation=validation,test=job['test'],c=clock['c'],clock_sha256=model['clock_sha256'],clock_path=model['clock_path']) + for role in ['learning','validation','test']: + f[role+'_nodes']=sum(geometry[case]['nodes'] for case in f[role]);f[role+'_edges']=sum(geometry[case]['directed_edges'] for case in f[role]) + assert not(set(f['learning'])&set(f['validation']) or set(f['learning'])&set(f['test']) or set(f['validation'])&set(f['test'])) + folds.append(f) + reps=[] + for key in ['nodes','directed_edges']: + order=sorted(folds[0]['learning'],key=lambda case:(geometry[case][key],case)) + for i in [0,len(order)//2,len(order)-1]: + if order[i] not in reps:reps.append(order[i]) + for rel in ['docs/phase2-h2-architecture-2026-11-17.md','references/2026-11-17/phase2-design/mold-h2-review-3541052d.md','docs/phase2-h2-execution-contract-2026-11-17.md']: + sources[rel]=sha(BASE/rel) + sources[str(acceptance.relative_to(BASE))]=sha(acceptance) + root=BASE/'artifacts/phase2-h2-pilot-2026-11-17';root.mkdir(exist_ok=False) + names=['phase2_h2_model','phase2_h2_data','phase2_h2_pilot_worker','phase2_h2_guard','phase2_h2_prepare'] + save(root/'contract.json',dict(version='phase2-h2-pilot-v1',approval='1794900702.867059',initial_pilot_approval='1794900338.137689',mode='PILOT_ONLY',full_after_cost_gate_authorized=True, + parts=old['parts']['grouped'],geometry=geometry,folds=folds,representatives=reps,code_sha256=closure(names),source_sha256=sources, + gpu_uuid='GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',gpu_index=0,worker_cpus=[0,1],guard_cpus=[2],gpu_seconds=7200,cpu_seconds=7200,pilot_setup_charge_seconds=60,future_setup_reserve_seconds=300, + rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30,versions=dict(torch=torch.__version__,numpy=np.__version__,cuda_build=torch.version.cuda), + model=dict(width=64,blocks=4,tokens_per_physical_component=64,attention_heads=4,edge_chunk=65536,node_chunk=8192,activation_checkpoint='each full block, non-reentrant',dtype='FP32',dropout=0), + pilot='learning-only normalization; small/median/max node and edge representatives two steps each; fresh seed0 complete fold0 epoch; complete validation forward timing with no targets', + optimizer='AdamW lr1e-3 betas.9/.999 eps1e-8 weight_decay1e-4 except biases/norm; clip1; one full-part step weighted Nparts*Ni/Ntotal', + schedule='1-based e1..10:1e-4+(e-1)*9e-4/9; e11..200:1e-5+(.001-.00001)*(1+cos(pi*(e-10)/190))/2; step after complete epoch', + normalization='learning-only original19 float64 parallel-Welford population std; zero std=>1; standardized network FP32; separate originalFP64 Vf skip', + limits_sampled=True,no_new_decodes=True,no_test_targets=True,no_retry=True,full_epochs=200,full_folds=5,full_seed=0,full_gpu_seconds_including_pilot=57600,full_max_parallel_gpus=4)) + print(json.dumps(dict(root=str(root),contract_sha256=sha(root/'contract.json'),representatives=reps,code_files=len(closure(names)),fold0_learning=len(folds[0]['learning']),validation=len(folds[0]['validation'])))) + +if __name__=='__main__':run(Path(sys.argv[1]).resolve()) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py new file mode 100644 index 000000000..396050685 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py @@ -0,0 +1,30 @@ +"""Labelled S1 amendment; original pilot and scientific sources remain immutable.""" +import json +from pathlib import Path +import numpy as np +import torch +from phase2_h1_worker_v2 import checked +from phase2_h2_data import config as pilot_config +BASE=Path('/home/frank14f/optane/MeltMind') +def read(p):return json.loads(Path(p).read_text()) +def config(root): + c=read(root/'contract.json') + if c['approval']!='1794901608.131369' or c['mode']!='H2_S1_FOLD0_LAST200':raise PermissionError('Unapproved S1 scope') + for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h) + old=pilot_config(Path(c['pilot_root'])) + if c['epochs']!=200 or c['seed']!=0 or c['fold']!=0 or c['total_h2_gpu_seconds']!=144000:raise ValueError('Frozen scientific recipe changed') + if c['prior_pilot_gpu_seconds']+c['gpu_seconds']>c['total_h2_gpu_seconds']:raise ValueError('Cumulative budget exceeds cap') + return c,old + +def gate(field_high,last_high): + return bool((field_high is not None and np.isfinite(field_high) and field_high<.005) or (last_high is not None and np.isfinite(last_high) and last_high<0)) + +def selected_checkpoint(completed_epochs): + if completed_epochs!=200:raise ValueError('Only final epoch200 may be evaluated') + return 'last-epoch200.pt' + +def stats_from_pilot(c,old): + stats=read(checked(c['standardization_path'],c['standardization_sha256'])) + if stats['parts']!=old['folds'][0]['learning'] or stats['nodes']!=old['folds'][0]['learning_nodes']:raise ValueError('Normalization learning domain mismatch') + if len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Invalid fixed normalization') + return stats diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py new file mode 100644 index 000000000..65b409d62 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py @@ -0,0 +1,83 @@ +"""Independent bounded S1 train guard; original pilot remains immutable.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase2_h2_s1_common import config +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(root): + c,_=config(root);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'s1.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single S1 launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds'])) + def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase2_h2_s1_worker.py',str(root)] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('S1_TIME_LIMIT') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='S1_TRAINING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + if p.exists():value=json.loads(p.read_text()) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(root) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and value.get('completed_epochs')!=200:reason('INCOMPLETE_EPOCH_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S1_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cumulative_h2_gpu_seconds=c['prior_pilot_gpu_seconds']+elapsed(),cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=False,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py new file mode 100644 index 000000000..46926da14 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py @@ -0,0 +1,20 @@ +"""Start the one published H2 S1 training after host/GPU duplicate checks.""" +import hashlib,json,pathlib,subprocess,sys,time,psutil +B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17' +proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published'] +for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p +contract=json.loads((root/'contract.json').read_text()) +for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p +for p in psutil.process_iter(['pid','cmdline']): + c=p.info['cmdline'] or [] + assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2' +assert not (root/'started.json').exists() and not (root/'service-launch.json').exists() +uuid=contract['gpu_uuid'] +raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True) +mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0 +unit='meltmind-phase2-h2-s1-20261117' +cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B), + '--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)] +r=subprocess.run(cmd,check=True,capture_output=True,text=True) +(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n') +print('SERVICE_LAUNCHED',unit,flush=True) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py new file mode 100644 index 000000000..f62aa6c24 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py @@ -0,0 +1,62 @@ +"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels.""" +import json,os,sys,time,traceback +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import seed,peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root): + c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stats=stats_from_pilot(c,old);save(out/'standardization.json',stats) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py new file mode 100644 index 000000000..6fe2caa29 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py @@ -0,0 +1,55 @@ +"""S2 identity/roles and cumulative GPU arithmetic; no fitting or target I/O.""" +import json,math +from pathlib import Path +from phase2_fixed_io import BASE +from phase2_h1_worker_v2 import checked,sha +from phase2_home_freeze_io import verify + +def read(p):return json.loads(Path(p).read_text()) +def execution_command(args): + return any(Path(x).name in {'phase2_h2_s2_queue.py','phase2_h2_s2_guard.py','phase2_h2_s2_worker.py','phase2_h2_s1_guard.py','phase2_h2_s1_worker.py','phase2_h2_s1_postfit.py'} for x in args) +def fold_view(pilot,fold): + if fold not in (1,2,3,4):raise PermissionError('Only new folds1..4; fold0 is immutable reuse') + f=pilot['folds'][fold] + if f['fold']!=fold:raise ValueError('Original fold identity') + roles=[set(f[k]) for k in ('learning','validation','test')] + if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') + return dict(pilot,folds=[f]) # Original preprocessing reads slot0; embedded fold identity remains actual fold. +def config(root): + c=read(root/'contract.json');verify(root,'implementation') + if c['approval']!='1794923791.467409' or c['total_h2_gpu_seconds']!=144000 or c['epochs']!=200 or c['seed']!=0 or [j['fold'] for j in c['jobs']]!=[1,2,3,4]:raise PermissionError('Fixed S2 scope') + if c['prior_gpu_seconds']!=18858.21490871499 or c['prediction_gpu_reserve']!=2400 or c['postfit_cpu_cap']!=3600 or c['new_fits']!=4:raise ValueError('Fixed prior ledger and bounded postfit reserve') + if len({j['gpu_uuid'] for j in c['jobs']})!=4:raise ValueError('Four distinct GPUs') + if c['train_gpu_cap']!=144000-c['prior_gpu_seconds']-c['prediction_gpu_reserve']:raise ValueError('GPU ledger identity') + for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) + pilot=read(c['pilot_contract']);prior=read(Path(c['s1_root'])/'post-terminal-cost.json') + import torch,numpy as np + if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Original scientific library versions required') + if prior['cumulative_h2_gpu_seconds']!=c['prior_gpu_seconds'] or not prior['within_cap']:raise ValueError('Original settled S1 cost') + for j in c['jobs']: + v=fold_view(pilot,j['fold']);f=v['folds'][0] + if j['learning_parts']!=len(f['learning']) or j['learning_nodes']!=f['learning_nodes'] or j['validation_nodes']!=f['validation_nodes']:raise ValueError('Original fold roster changed') + return c,pilot + +def projection(c,observations): + if set(observations)!={1,2,3,4}:raise ValueError('All four first epochs required') + rows=[] + for j in c['jobs']: + v=observations[j['fold']];epoch=v['epoch'];charged=v['charged_gpu_seconds'];train=v['max_train_epoch_seconds'];validation=max(v.get('max_validation_seconds',0),j['validation_seconds_bound']) + if not isinstance(epoch,int) or not 1<=epoch<=200 or not all(math.isfinite(x) and x>=0 for x in [charged,train,validation]) or train<=0:raise ValueError('Invalid timing evidence') + future=1.25*((200-epoch)*train+(20-epoch//10)*validation)+60 + estimate=charged+future;rows.append(dict(fold=j['fold'],completed_epochs=epoch,charged_gpu_seconds=charged,max_train_epoch_seconds=train,validation_seconds_bound=validation,projected_new_gpu_seconds=estimate,job_cap=j['gpu_seconds'],within_job_cap=estimate<=j['gpu_seconds'])) + total=c['prior_gpu_seconds']+sum(r['projected_new_gpu_seconds'] for r in rows)+c['prediction_gpu_reserve'] + passed=total<=c['total_h2_gpu_seconds'] and all(r['within_job_cap'] for r in rows) + return dict(status='PASS' if passed else 'STOP_BUDGET_PROJECTION',rows=rows,prior_gpu_once=c['prior_gpu_seconds'],prediction_gpu_reserve=c['prediction_gpu_reserve'],projected_total_gpu_seconds=total,total_cap=c['total_h2_gpu_seconds'],safety_factor=1.25,no_retry=True) + +def first_epoch_release(root,contract_sha): + if (root/'global-stop.json').exists():raise RuntimeError('Global S2 stop before next epoch') + p=root/'first-epoch-projection.json' + if not p.exists():return False + d=read(p) + if d.get('contract_sha256')!=contract_sha or d.get('status')!='PASS':raise RuntimeError('First-epoch budget gate rejected') + return True + +def confirmed(high,parts,components): + return bool(parts==552 and components==204 and high is not None and math.isfinite(high) and high<0) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py new file mode 100644 index 000000000..61752aa4a --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py @@ -0,0 +1,86 @@ +"""Independent bounded S2 fold guard; shared global stop and immutable S1.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase2_h2_s2_common import config +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(queue,fold_id): + c,_=config(queue);job=c['jobs'][fold_id-1];c=dict(c,**job);root=queue/'folds'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'fold.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single S2 fold launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds'])) + def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase2_h2_s2_worker.py',str(queue),str(fold_id)] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('S2_FOLD_TIME_LIMIT') + if (queue/'global-stop.json').exists():reason('S2_GLOBAL_STOP') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='S2_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + try: + if p.exists():value=json.loads(p.read_text()) + except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc))) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(queue) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and (value.get('completed_epochs')!=200 or value.get('optimizer_steps')!=200*c['learning_parts'] or value.get('fold')!=fold_id):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S2_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=True,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py new file mode 100644 index 000000000..54be0642d --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py @@ -0,0 +1,51 @@ +"""Metadata and hash preparation; no target arrays, GPU workload or launch.""" +import json,time,shutil +from pathlib import Path +import psutil +from phase2_fixed_io import BASE +from phase2_h1_worker_v2 import save,sha,checked +from phase2_home_freeze_io import freeze,verify +from phase2_h2_prepare import closure +from phase2_h2_s2_common import config,execution_command + +def read(p):return json.loads(Path(p).read_text()) +def main(): + root=Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17') + if root.exists():raise FileExistsError('Existing S2 root; inspect, do not duplicate') + active=[p.info for p in psutil.process_iter(['pid','cmdline']) if execution_command(p.info['cmdline'] or [])] + if active:raise RuntimeError('Existing H2 task') + s1=Path('/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17');verify(s1,'terminal');verify(s1,'predictions');old=read(s1/'contract.json');cost=read(s1/'post-terminal-cost.json');done=read(s1/'complete.json');pilotpath=BASE/'artifacts/phase2-h2-pilot-2026-11-17/contract.json';pilot=read(pilotpath);aroot=BASE/'artifacts/phase2-b13-e1b-2026-11-17';a=read(aroot/'contract.json');h=read(BASE/'artifacts/phase2-h1-execution-2026-11-17/contract.json') + if done['status']!='COMPLETE' or not done['s2_eligible_by_original_gate'] or not cost['within_cap'] or cost['cumulative_h2_gpu_seconds']!=18858.21490871499:raise ValueError('Original S1 gate or ledger') + pred=read(s1/'predictions-manifest.json')['entries'] + if len(pred)!=121:raise ValueError('Original121 predictions') + for e in pred:checked(e['path'],e['sha256']) + test=[case for f in pilot['folds'] for case in f['test']];by={r['case']:r for r in a['parts']} + if len(test)!=552 or len(set(test))!=552 or len({by[k]['component_id'] for k in test})!=204:raise ValueError('Fixed five-fold552/204') + for f in pilot['folds']: + af=a['folds'][f['fold']];hj=next(j for j in h['jobs'] if j['protocol']=='grouped' and j['pool']==100 and j['fold']==f['fold']) + if {r['case'] for r in af['rows']['learning']}!=set(f['learning']) or {r['case'] for r in af['rows']['test']}!=set(f['test']) or hj['baseline_model']!=af['A_baseline']:raise ValueError('Original H2/A model or fold role mismatch') + groups={role:{by[k]['component_id'] for k in f[role]} for role in ['learning','validation','test']} + if groups['test']&(groups['learning']|groups['validation']) or groups['learning']&groups['validation']:raise ValueError('Component leakage') + originaltrain=BASE/'artifacts/phase2-h2-s1-2026-11-17';history=[originaltrain/'worker/history'/f'{e:03d}.json' for e in range(10,201,10)];valmax=max(read(p)['validation_seconds'] for p in history) + uuids=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'];traincap=144000-cost['cumulative_h2_gpu_seconds']-2400;jobs=[];f0=pilot['folds'][0] + for fold in range(1,5): + f=pilot['folds'][fold];ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation'])) + jobs.append(dict(fold=fold,gpu_uuid=uuids[fold-1],worker_cpus=[2*(fold-1),2*(fold-1)+1],guard_cpus=[fold+7],gpu_seconds=traincap/4,cpu_seconds=traincap/4,setup_gpu_seconds=60,learning_parts=len(f['learning']),learning_nodes=f['learning_nodes'],validation_nodes=f['validation_nodes'],validation_seconds_bound=valmax*ratio,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)) + sources={str(BASE/p):h for p,h in read(originaltrain/'contract.json')['source_sha256'].items()} + for p in [s1/'contract.json',s1/'complete.json',s1/'post-terminal-cost.json',s1/'score/result.json',s1/'predictions-manifest.json',s1/'terminal-git-freeze.json',s1/'terminal-authorization.json',s1/'predictions-git-freeze.json',s1/'predictions-authorization.json',Path(old['checkpoint']),pilotpath,aroot/'contract.json',aroot/'predictions-manifest.json',aroot/'score/result.json',*history]:sources[str(p)]=sha(p) + for f in pilot['folds']:sources[f['clock_path']]=f['clock_sha256'] + for p,digest in sources.items():checked(p,digest) + codes={str(BASE/p):h for p,h in closure(['phase2_h2_s2_common','phase2_h2_s2_worker','phase2_h2_s2_guard','phase2_h2_s2_queue','phase2_h2_s2_prepare','test_phase2_h2_s2']).items()} + for p,h in read(originaltrain/'contract.json')['code_sha256'].items():checked(BASE/p,h);codes[str(BASE/p)]=h + doc=BASE/'docs/phase2-h2-s2-execution-contract-2026-11-17.md';codes[str(doc)]=sha(doc) + root.mkdir();fold0=dict(checkpoint=old['checkpoint'],checkpoint_sha256=old['checkpoint_sha256'],prediction_manifest=str(s1/'predictions-manifest.json'),prediction_manifest_sha256=sha(s1/'predictions-manifest.json'),score=str(s1/'score/result.json'),score_sha256=sha(s1/'score/result.json'),terminal=verify(s1,'terminal')['commit'],no_reexecution=True) + c=dict(version='H2-S2-four-fold-training-v1',approval='1794923791.467409',correction='1794923933.456449',epochs=200,seed=0,new_fits=4,s1_root=str(s1),pilot_contract=str(pilotpath),fold0_reuse=fold0,prior_gpu_seconds=cost['cumulative_h2_gpu_seconds'],total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,train_gpu_cap=traincap,postfit_cpu_cap=3600,jobs=jobs,coordinator_cpus=[12,13],aggregate_rss_bytes=64*2**30,disk_bytes=80*2**30,source_sha256=sources,code_sha256=codes,scoring=dict(parts=552,components=204,bootstrap_replicates=2000,bootstrap_seed=2026111408,confirmation='pooled field H2-A CI upper<0 with complete552/204',B13_and_last_and_tau='report_only',new_exact_semantic_PASS_required=True,S1_PASS_reuse=False,fold0_score_hash_reuse=True),stop_after='TRAIN_COMPLETE_POSTFIT_PENDING',no_retry=True,data_sufficiency_started=False) + save(root/'contract.json',c) + for i in [1,2]:shutil.copyfile('/tmp/h2-s2-synthetic-attempt'+str(i)+'.log',root/('synthetic-attempt'+str(i)+'.log')) + if not (root/'synthetic-attempt2.log').read_text().rstrip().endswith('OK'):raise ValueError('Synthetic acceptance absent') + save(root/'acceptance.json',dict(status='PASS',tests=16,elapsed_seconds=.051,command='python -m unittest tools.test_phase2_h2_s2 -v',actual_log_sha256=sha(root/'synthetic-attempt2.log'),first_fixture_failure_retained=True,prelaunch_contract_path_fixed=True,scientific_workloads=0,scope='CPU synthetic/AST/role/budget and two guard lifecycle tests with fake supervisor/GPU probe; no real CUDA or targets',unix=time.time())) + gate=Path('/home/frank14f/MeltMind-runs/phase2-gate-outcome-calibration-v2-2026-11-17');g=read(gate/'contract.json');gprog=read(gate/'live-progress.json');gatepins={p:sha(BASE/p)==h for p,h in g['code_sha256'].items()};assert all(gatepins.values()) + save(root/'preparation-audit.json',dict(status='PREPARED_NOT_LAUNCHED',unix=time.time(),h2_active_processes=active,original_fold0_predictions_hash_checked=121,original_A_roles_models_checked=5,test_partition552_204=True,test_target_values_read=False,validation_timing_source_max=valmax,gate_progress=gprog,gate_original_total_cpu=72000,gate_geometry_cpu=68400,gate_pins_verified=all(gatepins.values()),gate_95000_ruling='1794919865.091919 /1794920747.468449; only after original budget stop and exit; all terminal hashes/old CPU once, no retries, new postfit identity review',gate_modified=False)) + freeze(root,'implementation',[root/'contract.json',root/'acceptance.json',root/'preparation-audit.json',root/'synthetic-attempt1.log',root/'synthetic-attempt2.log',*[Path(p) for p in codes]]);config(root) + print(json.dumps(dict(root=str(root),implementation=verify(root,'implementation')['commit'],contract_sha256=sha(root/'contract.json'),code_files=len(codes),jobs=[dict(fold=j['fold'],learning_parts=j['learning_parts'],gpu_seconds=j['gpu_seconds']) for j in jobs],launched=False))) +if __name__=='__main__':main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py new file mode 100644 index 000000000..dd7278164 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py @@ -0,0 +1,100 @@ +"""One four-fold S2 queue, global GPU budget, first-epoch gate; stops before postfit.""" +import fcntl,json,os,signal,subprocess,sys,time +from pathlib import Path +import psutil +from phase2_h2_s2_common import config,read,projection,execution_command +from phase2_h1_worker_v2 import save,sha,checked +from phase2_home_freeze_io import freeze,verify +from s2_recovery_gpu_supervisor import tree_usage,disk_usage + +def run(root): + c,pilot=config(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single S2 queue, no retry') + for p in psutil.process_iter(['pid','cmdline']): + if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing H2 execution process') + os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];errors=[];timings={};projection_round=None + def stop(reason): + if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True)) + def charged(): + return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values()) + def terminate(v): + if v.get('stop_sent') is None: + v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic() + elif time.monotonic()-v['stop_sent']>20: + try: + parent=psutil.Process(v['process'].pid);children=parent.children(recursive=True) + for p in reversed(children): + try:p.kill() + except psutil.NoSuchProcess:pass + parent.kill() + except psutil.NoSuchProcess:pass + def collect(fold,v): + p=v['process'];rc=p.poll() + if rc is None:return False + elapsed=60+time.monotonic()-v['start'];receipt_path=root/'folds'/str(fold)/'complete.json';d=read(receipt_path) if receipt_path.exists() else None + cost=max(elapsed,d['gpu_seconds_charged'] if d else 0);ok=rc==0 and d is not None and d['status']=='PASS' + row=dict(fold=fold,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,coordinator_observed_seconds=elapsed,guard_gpu_seconds=d['gpu_seconds_charged'] if d else None,complete_sha256=sha(receipt_path) if d else None,receipt_present=d is not None) + terminal[fold]=row;v['log'].close();del active[fold];save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()),prior_gpu_once=c['prior_gpu_seconds'])) + if not ok:stop('FOLD_'+str(fold)+'_FAILED_OR_MISSING_TERMINAL') + return True + def observe(fold): + history=root/'folds'/str(fold)/'worker/history';rows=[] + for p in sorted(history.glob('*.json')): + d=read(p) + if d['epoch_parts']!=c['jobs'][fold-1]['learning_parts'] or d['epoch_nodes']!=c['jobs'][fold-1]['learning_nodes'] or d['optimizer_steps']!=d['epoch']*c['jobs'][fold-1]['learning_parts']:raise ValueError('Incomplete timing epoch evidence') + rows.append(d) + if not rows:return None + cost=terminal[fold]['gpu_seconds_charged'] if fold in terminal else 60+time.monotonic()-active[fold]['start'] + return dict(epoch=rows[-1]['epoch'],charged_gpu_seconds=cost,max_train_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) + old={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]} + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),folds=[1,2,3,4],prior_gpu_once=c['prior_gpu_seconds'],prediction_gpu_reserve=c['prediction_gpu_reserve'],postfit_started=False)) + try: + for job in c['jobs']: + fold=job['fold'];d=root/'folds'/str(fold);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic() + command=[sys.executable,'tools/phase2_h2_s2_guard.py',str(root),str(fold)];p=subprocess.Popen(command,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) + active[fold]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(fold=fold,guard_pid=p.pid,command=command,unix=time.time(),gpu_uuid=job['gpu_uuid'],setup_gpu_seconds=60)) + while active: + for fold,v in list(active.items()):collect(fold,v) + for reason in requested:stop(reason) + amount=charged();_,rss,_=tree_usage(os.getpid());used=disk_usage(root) + if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_GPU_RESERVE_BOUNDARY') + if rss>c['aggregate_rss_bytes']:stop('AGGREGATE_RSS_LIMIT') + if used>c['disk_bytes']:stop('TOTAL_OUTPUT_DISK_LIMIT') + if not (root/'global-stop.json').exists(): + timings={f:observe(f) for f in range(1,5)} + if all(v is not None for v in timings.values()): + current_round=min(v['epoch'] for v in timings.values())//10 + if projection_round is None or current_round>projection_round: + d=projection(c,timings);d.update(contract_sha256=sha(root/'contract.json'),unix=time.time());save(root/('first-epoch-projection.json' if projection_round is None else 'latest-cost-projection.json'),d) + projection_round=current_round + if d['status']!='PASS':stop('PROJECTED_GPU_BUDGET_EXCEEDED') + if (root/'global-stop.json').exists(): + for v in active.values():terminate(v) + save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'S2_TRAINING',unix=time.time(),active=[dict(fold=f,guard_pid=v['process'].pid) for f,v in active.items()],terminal=list(terminal.values()),new_gpu_seconds_charged=amount,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],train_gpu_cap=c['train_gpu_cap'],epoch_counts={str(f):(x['epoch'] if x else 0) for f,x in timings.items()},rss_bytes=rss,disk_bytes=used)) + if active:time.sleep(1) + if (root/'global-stop.json').exists() or len(terminal)!=4 or any(r['status']!='PASS' for r in terminal.values()):raise RuntimeError('S2 training incomplete; originals retained, no retries') + config(root);models=[] + for j in c['jobs']: + fold=j['fold'];w=root/'folds'/str(fold)/'worker';r=read(w/'result.json');checkpoint=checked(r['checkpoint'],r['checkpoint_sha256']);stats=read(checked(w/'standardization.json',r['standardization_sha256'])) + if r['fold']!=fold or r['completed_epochs']!=200 or r['optimizer_steps']!=200*j['learning_parts'] or r['test_targets_read'] is not False or stats['parts']!=pilot['folds'][fold]['learning']:raise ValueError('Final checkpoint/normalization role identity') + models.append(dict(fold=fold,checkpoint=str(checkpoint),checkpoint_sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],worker_result_sha256=sha(w/'result.json'))) + amount=charged() + if amount>c['train_gpu_cap'] or c['prior_gpu_seconds']+amount+c['prediction_gpu_reserve']>144000:raise RuntimeError('Final aggregate GPU limit') + save(root/'models-terminal.json',dict(models=models,fold0_reused_by_hash=c['fold0_reuse'],new_fits=4,test_targets_read=False,postfit_started=False));freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['checkpoint']) for m in models],*[Path(m['standardization']) for m in models]]) + save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,new_fits=4,postfit_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json']) + except BaseException as exc: + stop(type(exc).__name__+': '+str(exc));errors.append(str(exc)) + while active: + for f,v in list(active.items()): + try: + if not collect(f,v):terminate(v) + except Exception as e: + errors.append(str(e));terminate(v) + if v['process'].poll() is not None: + terminal[f]=dict(fold=f,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start'],returncode=v['process'].returncode);v['log'].close();del active[f] + if active:time.sleep(.5) + amount=charged();save(root/'complete.json',dict(status='STOPPED',reason=str(exc),errors=errors,terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_retry=True));return 1 + finally: + for s,h in old.items():signal.signal(s,h) + return 0 +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py new file mode 100644 index 000000000..a475310c8 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py @@ -0,0 +1,67 @@ +"""One new S2 fold, same frozen seed0 recipe/200epochs; no test prediction or target reads.""" +import json,os,sys,time,traceback +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import selected_checkpoint +from phase2_h2_s2_common import config,fold_view,first_epoch_release +from phase2_h2_data import standardization +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import seed,peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root,fold_id): + c,pilot=config(root);old=fold_view(pilot,fold_id);job=c['jobs'][fold_id-1];out=root/'folds'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256'])) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch==1: + stage='WAIT_ALL_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes)) + while not first_epoch_release(root,sha(root/'contract.json')):time.sleep(.5) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py new file mode 100644 index 000000000..4fb7783e1 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py @@ -0,0 +1,29 @@ +"""Local immutable barriers for explicitly approved /home run roots.""" +import fcntl,json,shutil,subprocess,hashlib +from pathlib import Path +from phase2_h1_worker_v2 import sha,checked,save +from phase2_local_freeze_io import STORE,LOCK,BRANCH,BASE +HOME_RUNS=Path('/home/frank14f/MeltMind-runs') +def key(path): + p=Path(path).resolve() + if p.is_relative_to(HOME_RUNS):return 'home-runs/'+str(p.relative_to(HOME_RUNS)) + return str(p.relative_to(BASE)) +def freeze(root,name,files): + with LOCK.open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + def git(*a):return subprocess.check_output(['git','-C',str(STORE),'-c','gc.auto=0',*a],text=True).strip() + if git('branch','--show-current')!=BRANCH or git('status','--porcelain'):raise RuntimeError('Local writer not clean') + hashes={};paths={} + for f in files: + f=Path(f).resolve();k=key(f);p=STORE/k;p.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(f,p);hashes[str(f)]=sha(f);paths[str(f)]=k + subprocess.run(['git','-C',str(STORE),'add','--',*paths.values()],check=True) + if git('diff','--cached','--name-only'):subprocess.run(['git','-C',str(STORE),'-c','user.name=MeltAstra','-c','user.email=meltrastra@local','commit','-q','-m','Immutable home run '+root.name+' '+name],check=True) + commit=git('rev-parse','HEAD') + for p,h in hashes.items(): + if hashlib.sha256(subprocess.check_output(['git','-C',str(STORE),'show',commit+':'+paths[p]])).hexdigest()!=h:raise ValueError('Git content mismatch') + save(root/(name+'-git-freeze.json'),dict(commit=commit,files=hashes,git_paths=paths,remote_published=False,local_repository=str(STORE))) + save(root/(name+'-authorization.json'),dict(contract_sha256=sha(root/'contract.json'),proof_sha256=sha(root/(name+'-git-freeze.json')))) +def verify(root,name): + a=json.loads((root/(name+'-authorization.json')).read_text());checked(root/'contract.json',a['contract_sha256']);p=checked(root/(name+'-git-freeze.json'),a['proof_sha256']);d=json.loads(p.read_text()) + for f,h in d['files'].items():checked(f,h) + return d diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py new file mode 100644 index 000000000..14a03cd7e --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py @@ -0,0 +1,35 @@ +"""6657 engineering-only local Git barriers; unchanged scientific proof verifier.""" +import fcntl,hashlib,json,shutil,subprocess +from pathlib import Path +from phase2_h1_worker_v2 import sha,save +BASE=Path('/home/frank14f/optane/MeltMind') +STORE=Path('/tmp/meltmind-phase2-local-fixed-20261117') +LOCK=Path('/tmp/meltmind-phase2-local-fixed-20261117.lock') +BRANCH='codex/phase2-local-fixed-20261117' +ENGINEERING=BASE/'artifacts/phase2-local-launch-2026-11-17/engineering-contract.json' + +def freeze(root,name,files): + with LOCK.open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + def git(*args):return subprocess.check_output(['git','-C',str(STORE),'-c','gc.auto=0',*args],text=True).strip() + if not (STORE/'.git').exists(): + STORE.mkdir(exist_ok=True) + if list(STORE.iterdir()):raise RuntimeError('Unrecognized local evidence store') + subprocess.run(['git','init','-q','-b',BRANCH,str(STORE)],check=True) + if git('branch','--show-current')!=BRANCH or git('status','--porcelain'):raise RuntimeError('Local barrier writer not clean') + hashes={} + extra=[] + if ENGINEERING.exists(): + e=json.loads(ENGINEERING.read_text());extra=[ENGINEERING,*[BASE/p for p in e['engineering_sha256']]] + for p,h in e['engineering_sha256'].items(): + if sha(BASE/p)!=h:raise RuntimeError('Engineering source changed: '+p) + for p in [*files,*extra]: + p=Path(p);rel=str(p.resolve().relative_to(BASE));dest=STORE/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest);hashes[rel]=sha(p) + subprocess.run(['git','-C',str(STORE),'add','--',*hashes],check=True) + if git('diff','--cached','--name-only'): + subprocess.run(['git','-C',str(STORE),'-c','user.name=MeltAstra','-c','user.email=meltrastra@local','commit','-q','-m','Phase2 local immutable '+root.name+' '+name],check=True) + commit=git('rev-parse','HEAD') + for rel,h in hashes.items(): + if hashlib.sha256(subprocess.check_output(['git','-C',str(STORE),'show',commit+':'+rel])).hexdigest()!=h:raise ValueError('Local Git byte mismatch') + save(root/(name+'-git-freeze.json'),dict(commit=commit,sha256=hashes,remote_published=False,local_repository=str(STORE),backend='6657_local_immutable_barrier',approval='1794906657.779289')) + save(root/(name+'-authorization.json'),dict(proof_sha256=sha(root/(name+'-git-freeze.json')),contract_sha256=sha(root/'contract.json'))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py new file mode 100644 index 000000000..267619a86 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py @@ -0,0 +1,65 @@ +"""Array-only residual interface; no I/O, model, labels, or clock fitting policy. + +The caller supplies a frozen, training-only clock. It also fixes the F precision +in the reviewed contract. Passing F explicitly avoids a hidden extra feature. +""" +import numpy as np + + +def learning_clock(end_ratios, component_ids): + """Mean over components of mean retained-tet-end/t_set per learning part. + + For the random v4 protocol each part can be its own unit, if the final + specification chooses part weighting. Caller enforces training roles. + """ + ratios = np.asarray(end_ratios, dtype=np.float64) + if ratios.ndim != 1 or not len(ratios) or len(component_ids) != len(ratios): + raise ValueError('Clock row count mismatch') + if not np.isfinite(ratios).all() or np.any(ratios <= 0): + raise ValueError('Invalid retained-tet endpoint ratio') + groups = {} + for group, ratio in zip(component_ids, ratios): + if not isinstance(group, str) or not group: + raise ValueError('Missing clock group identity') + groups.setdefault(group, []).append(float(ratio)) + return float(np.mean([np.mean(v) for v in groups.values()])) + + +def prior(cdf, clock): + f = np.asarray(cdf) + if f.ndim != 1 or f.dtype.kind != 'f' or not len(f): + raise ValueError('Expected nonempty floating-point CDF vector') + if not np.isfinite(f).all() or np.any(f < 0) or np.any(f > 1): + raise ValueError('CDF outside [0,1] or nonfinite') + if not np.isscalar(clock) or not np.isfinite(clock) or clock <= 0: + raise ValueError('Clock must be a finite positive frozen scalar') + return np.asarray(f, dtype=np.float64)*float(clock) + + +def residual_target(target_star, cdf, clock, *, fit_dtype): + y = np.asarray(target_star, dtype=np.float64) + base = prior(cdf, clock) + if y.shape != base.shape or not np.isfinite(y).all() or np.any(y < 0): + raise ValueError('Invalid original normalized arrival target') + dtype = np.dtype(fit_dtype) + if dtype not in (np.dtype('float32'), np.dtype('float64')): + raise ValueError('Fit precision must be explicitly float32 or float64') + out = np.asarray(y-base, dtype=dtype) + if not np.isfinite(out).all(): + raise ValueError('Residual overflow at fit precision') + # Negative residuals are valid. Never clip or impose a positivity loss here. + return out + + +def decode_seconds(predicted_residual, cdf, clock, t_set): + residual = np.asarray(predicted_residual, dtype=np.float64) + base = prior(cdf, clock) + if residual.shape != base.shape or not np.isfinite(residual).all(): + raise ValueError('Invalid predicted residual') + if not np.isscalar(t_set) or not np.isfinite(t_set) or t_set <= 0: + raise ValueError('Invalid pre-solve seconds scale') + out = (base+residual)*float(t_set) + if not np.isfinite(out).all(): + raise ValueError('Decoded prediction overflow') + # Keep negative predictions visible to scoring/diagnostics; no silent clamp. + return out diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py new file mode 100644 index 000000000..d67b00ced --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py @@ -0,0 +1,116 @@ +"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates.""" +import math +from pathlib import Path +from phase2_h1_worker_v2 import checked,sha +from phase2_home_freeze_io import verify +from track_a_h2_common import read + +GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'] + + +def execution_command(args): + return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py'} for x in args) + + +def fold_view(pilot,fold): + if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds') + f=pilot['folds'][fold] + if f['fold']!=fold:raise ValueError('Actual original fold identity') + roles=[set(f[k]) for k in ('learning','validation','test')] + if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') + return dict(pilot,folds=[f]) + + +def config(root): + c=read(root/'contract.json');verify(root,'implementation') + if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope') + if c['dispatch_approval']!='1794983024.445549' or c['approval']!='1794982860.347839' or c['gpu_cap']!=118800 or c['train_gpu_cap']!=111600 or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000:raise PermissionError('Frozen seed1 and phase3 ceilings') + for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) + pilot=read(c['pilot_contract']) + import numpy as np,torch + if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions') + if sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Partitioned training budget') + if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve') + tests=[] + for j in c['jobs']: + f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test']) + for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']: + if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k) + if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved GPU assignment and all epochs') + if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition') + return c,pilot + + +def project_cell(job,observation): + e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged'] + validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0)) + if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing') + future=1.25*((200-e)*train+(20-e//10)*validation)+120 + value=charged+future + return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,within_cap=value<=job['gpu_seconds']) + + +def project_total(c,estimates,terminal,active_charges=None): + active_charges=active_charges or {};ids={j['id'] for j in c['jobs']} + if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job') + rows=[];total=c['postfit_gpu_reserve'] + for j in c['jobs']: + i=j['id'];v=terminal[i]['gpu_seconds_charged'] if i in terminal else max(active_charges.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',j['gpu_seconds'])) + if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger') + total+=v;rows.append(dict(id=i,projected_gpu_seconds=v,within_cap=v<=j['gpu_seconds'])) + ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['within_cap'] for r in rows) + return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve']) + + +def first_epoch_release(root,jobid): + if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop') + p=root/'jobs'/str(jobid)/'first-epoch-projection.json' + if not p.exists():return False + r=read(p) + if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected') + return True + + +def dispatchable(pending,active,available): + occupied={j['gpu_uuid'] for j in active} + if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU') + free=[u for u in GPUS if u in available and u not in occupied] + return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)] + + +def idle_eligible_gpus(eligible,query=None): + if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU') + if query is None: + from s2_precision_continuation_guard import device_usage + query=device_usage + idle=set();observations=[] + for u in GPUS: + if u not in eligible:continue + mem,util=query(u) + if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy') + ok=mem<=128*2**20 and util==0 + observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok)) + if ok:idle.add(u) + return idle,observations + + +def assigned_job(root,c,index,digest): + p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p) + if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity') + return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid']) + + +def available_gpus(c): + import psutil + available={GPUS[0]} + item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root']) + names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []} + if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}: + available.add(GPUS[1]) + if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}: + # GPU2 stays reserved for the already-prioritized Track A postfit until its terminal. + post=Path(c['track_a_postfit_root']) + available.add(GPUS[3]) + if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}: + available.add(GPUS[2]) + return available diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py new file mode 100644 index 000000000..7bb22ba80 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py @@ -0,0 +1,127 @@ +"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates.""" +import math +from pathlib import Path +from phase2_h1_worker_v2 import checked,sha +from phase2_home_freeze_io import verify +from track_a_h2_common import read + +GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'] + + +def execution_command(args): + return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py'} for x in args) + + +def fold_view(pilot,fold): + if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds') + f=pilot['folds'][fold] + if f['fold']!=fold:raise ValueError('Actual original fold identity') + roles=[set(f[k]) for k in ('learning','validation','test')] + if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') + return dict(pilot,folds=[f]) + + +def config(root): + c=read(root/'contract.json');verify(root,'implementation') + if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope') + if c['dispatch_approval']!='1794983024.445549' or c['approval']!='1794984996.938059' or c['accounting_approval']!='1794985731.591689' or c['gpu_cap']!=167200-c['old_seed1_gpu_once'] or c['train_gpu_cap']!=160000-c['old_seed1_gpu_once'] or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000:raise PermissionError('Frozen seed1 and phase3 ceilings') + for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) + old=Path(c['original_seed1_root']) + if str(old)!='/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18' or root.resolve()!=old.with_name('phase3-seed1-continuation-2026-11-18'):raise PermissionError('Sole labelled continuation root') + if c['continuation_number']!=1 or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed continuation scope') + d=read(old/'complete.json');oc=read(old/'contract.json') + if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or [e['id'] for e in d['entries']]!=[0,1]:raise PermissionError('Immutable original stop') + spent={e['id']:e['gpu_seconds_charged'] for e in d['entries']} + if c['old_seed1_gpu_once']!=d['training_gpu_seconds'] or sum(spent.values())!=c['old_seed1_gpu_once']:raise ValueError('Old seed1 charged exactly once') + if c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+c['old_seed1_gpu_once']:raise ValueError('Phase3 original item plus old seed1 once') + if c['cumulative_seed1_gpu_cap']!=167200 or c['cumulative_fold_gpu_cap']!=32000:raise ValueError('Final cumulative ceilings') + if any(j['gpu_seconds']!=32000-spent.get(j['id'],0) or j['cpu_seconds']!=j['gpu_seconds'] for j in c['jobs']):raise ValueError('Per-fold remaining caps') + if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged') + pilot=read(c['pilot_contract']) + import numpy as np,torch + if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions') + if sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Partitioned training budget') + if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve') + tests=[] + for j in c['jobs']: + f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test']) + for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']: + if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k) + if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved GPU assignment and all epochs') + if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition') + return c,pilot + + +def project_cell(job,observation): + e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged'] + validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0)) + if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing') + future=1.25*((200-e)*train+(20-e//10)*validation)+120 + value=charged+future + return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,within_cap=value<=job['gpu_seconds']) + + +def project_total(c,estimates,terminal,active_charges=None): + active_charges=active_charges or {};ids={j['id'] for j in c['jobs']} + if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job') + rows=[];total=c['postfit_gpu_reserve'] + for j in c['jobs']: + i=j['id'];v=terminal[i]['gpu_seconds_charged'] if i in terminal else max(active_charges.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',j['gpu_seconds'])) + if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger') + total+=v;rows.append(dict(id=i,projected_gpu_seconds=v,within_cap=v<=j['gpu_seconds'])) + ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['within_cap'] for r in rows) + return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve']) + + +def first_epoch_release(root,jobid): + if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop') + p=root/'jobs'/str(jobid)/'first-epoch-projection.json' + if not p.exists():return False + r=read(p) + if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected') + return True + + +def dispatchable(pending,active,available): + occupied={j['gpu_uuid'] for j in active} + if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU') + free=[u for u in GPUS if u in available and u not in occupied] + return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)] + + +def idle_eligible_gpus(eligible,query=None): + if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU') + if query is None: + from s2_precision_continuation_guard import device_usage + query=device_usage + idle=set();observations=[] + for u in GPUS: + if u not in eligible:continue + mem,util=query(u) + if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy') + ok=mem<=128*2**20 and util==0 + observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok)) + if ok:idle.add(u) + return idle,observations + + +def assigned_job(root,c,index,digest): + p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p) + if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity') + return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid']) + + +def available_gpus(c): + import psutil + available={GPUS[0]} + item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root']) + names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []} + if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}: + available.add(GPUS[1]) + if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}: + # GPU2 stays reserved for the already-prioritized Track A postfit until its terminal. + post=Path(c['track_a_postfit_root']) + available.add(GPUS[3]) + if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}: + available.add(GPUS[2]) + return available diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py new file mode 100644 index 000000000..887ab337e --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py @@ -0,0 +1,88 @@ +"""Independent bounded SEED1 fold guard; shared global stop and immutable S1.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase3_seed1_continuation_common import config,assigned_job +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(queue,fold_id,assignment_sha): + c,_=config(queue);job=assigned_job(queue,c,fold_id,assignment_sha);c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-seed1-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single SEED1 fold launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds'])) + def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase3_seed1_continuation_worker.py',str(queue),str(fold_id),assignment_sha] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('SEED1_FOLD_TIME_LIMIT') + if (queue/'global-stop.json').exists():reason('SEED1_GLOBAL_STOP') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='SEED1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START') + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + try: + if p.exists():value=json.loads(p.read_text()) + except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc))) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(queue) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('fold')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'SEED1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,seed1_started=True,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py new file mode 100644 index 000000000..ca316e112 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py @@ -0,0 +1,212 @@ +"""Five original grouped folds on released GPUs; train and freeze only, never score.""" +import fcntl +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +import psutil +from phase2_gate_outcome_review import validate_review +from phase2_h1_worker_v2 import checked, save, sha +from phase2_home_freeze_io import freeze +from s2_recovery_gpu_supervisor import disk_usage, tree_usage +from phase3_seed1_continuation_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus + + +def run(root): + c, _ = config(root) + validate_review(root) + lock = (root / 'queue.lock').open('a') + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (root / 'started.json').exists(): + raise FileExistsError('One Seed1 attempt; no retry') + for p in psutil.process_iter(['pid', 'cmdline']): + if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []): + raise RuntimeError('Existing Seed1 execution') + os.sched_setaffinity(0, c['coordinator_cpus']) + os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' + start = time.monotonic() + cpu = time.process_time() + active, terminal, estimates, last_epoch = {}, {}, {}, {} + pending = list(c['jobs']) + requested = [] + + def stop(reason): + if not (root / 'global-stop.json').exists(): + save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True)) + + def active_charges(): + return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + for i, v in active.items()} + + def charged(): + return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values()) + + def terminate(v): + if v.get('stop_sent') is None: + if v['process'].poll() is None: + v['process'].send_signal(signal.SIGTERM) + v['stop_sent'] = time.monotonic() + elif time.monotonic() - v['stop_sent'] > 20: + try: + parent = psutil.Process(v['process'].pid) + for p in reversed(parent.children(recursive=True)): + try: + p.kill() + except psutil.NoSuchProcess: + pass + parent.kill() + except psutil.NoSuchProcess: + pass + + def collect(i, v): + rc = v['process'].poll() + if rc is None: + return False + p = root / 'jobs' / str(i) / 'complete.json' + r = read(p) if p.exists() else None + observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + ok = (rc == 0 and r is not None and r['status'] == 'PASS' + and not r.get('monitor_errors') and r.get('worker') is not None + and not r['worker'].get('monitor_errors')) + terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc, + gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0), + guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json')) + v['log'].close() + del active[i] + save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()), + settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()))) + if not ok: + stop('JOB_' + str(i) + '_FAILED') + return True + + def dispatch(): + eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()} + idle,observations=idle_eligible_gpus(eligible) + observed_unix=time.time() + save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations)) + for j in dispatchable(pending, [v['job'] for v in active.values()], idle): + i = j['id'] + d = root / 'jobs' / str(i) + d.mkdir(parents=True, exist_ok=False) + log = (d / 'entry.log').open('x') + begin = time.monotonic() + save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix)) + cmd = [sys.executable, 'tools/phase3_seed1_continuation_guard.py', str(root), str(i), sha(d/'assignment.json')] + try: + p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + except BaseException: + log.close() + raise + active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j) + pending.remove(c['jobs'][i]) + save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid'])) + + handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n))) + for s in (signal.SIGTERM, signal.SIGINT)} + save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5, + contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'), + gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False)) + try: + while active or pending: + for i, v in list(active.items()): + collect(i, v) + for reason in requested: + stop(reason) + amount = charged() + _, rss, _ = tree_usage(os.getpid()) + if amount >= c['train_gpu_cap'] - 80: + stop('CUMULATIVE_TRAIN_GPU_BOUNDARY') + if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']: + stop('AGGREGATE_MEMORY_DISK_LIMIT') + if not (root / 'global-stop.json').exists(): + for i in active: + files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json')) + rows = [read(p) for p in files] + if not rows: + continue + epoch = rows[-1]['epoch'] + # Recalculate every epoch: waiting, setup and observed cost stay charged. + if epoch > last_epoch.get(i, 0): + observation = dict(epoch=epoch, charged=active_charges()[i], + max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows), + max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) + cell = project_cell(c['jobs'][i], observation) + estimates[i] = cell + projection = dict(**project_total(c, estimates, terminal, active_charges()), + cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time()) + name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json' + save(root / 'jobs' / str(i) / name, projection) + save(root / 'latest-cost-projection.json', projection) + last_epoch[i] = epoch + if projection['status'] != 'PASS': + stop('PROJECTED_SEED1_BUDGET') + if not (root / 'global-stop.json').exists(): + dispatch() + if (root / 'global-stop.json').exists(): + for v in active.values(): + terminate(v) + save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists() + else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid) + for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending], + charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'])) + if (root / 'global-stop.json').exists() and not active: + break + if active or pending: + time.sleep(1) + if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()): + raise RuntimeError('Seed1 training incomplete') + config(root) + models = [] + for j in c['jobs']: + w = root / 'jobs' / str(j['id']) / 'worker' + r = read(w / 'result.json') + stats = read(checked(w / 'standardization.json', r['standardization_sha256'])) + if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200 + or stats['parts'] != j['learning'] or r['test_targets_read']): + raise ValueError('Terminal model identity') + checked(r['checkpoint'], r['checkpoint_sha256']) + models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'], + standardization=str(w / 'standardization.json'), + standardization_sha256=r['standardization_sha256'])) + if charged() > c['train_gpu_cap']: + raise ValueError('Terminal cumulative cost') + save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False)) + freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json', + *[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]]) + save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()), + training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'], + coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start, + phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(), + no_retry=True, test_targets_read=False, postfit_started=False)) + freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json']) + return 0 + except BaseException as exc: + stop(type(exc).__name__ + ': ' + str(exc)) + while active: + for i, v in list(active.items()): + try: + if not collect(i, v): + terminate(v) + except Exception: + terminate(v) + if v['process'].poll() is not None: + terminal[i] = dict(id=i, status='MALFORMED_TERMINAL', + gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']) + v['log'].close() + del active[i] + if active: + time.sleep(.5) + save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()), + never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(), + postfit_started=False, no_retry=True)) + return 1 + finally: + for s, handler in handlers.items(): + signal.signal(s, handler) + + +if __name__ == '__main__': + raise SystemExit(run(Path(sys.argv[1]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py new file mode 100644 index 000000000..3b6c13652 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py @@ -0,0 +1,67 @@ +"""One new S2 fold, same frozen recipe with seed1/200epochs; no test prediction or target reads.""" +import json,os,sys,time,traceback,random +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import selected_checkpoint +from phase3_seed1_continuation_common import config,fold_view,first_epoch_release,assigned_job +from phase2_h2_data import standardization +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root,fold_id,assignment_sha): + c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256'])) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(1);np.random.seed(1);torch.manual_seed(1);torch.cuda.manual_seed_all(1) + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=1,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([1,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch==1: + stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes)) + while not first_epoch_release(root,fold_id):time.sleep(.5) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=1,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py new file mode 100644 index 000000000..bca124d55 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py @@ -0,0 +1,88 @@ +"""Independent bounded SEED1 fold guard; shared global stop and immutable S1.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase3_seed1_common import config,assigned_job +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(queue,fold_id,assignment_sha): + c,_=config(queue);job=assigned_job(queue,c,fold_id,assignment_sha);c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-seed1-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single SEED1 fold launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds'])) + def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase3_seed1_worker.py',str(queue),str(fold_id),assignment_sha] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('SEED1_FOLD_TIME_LIMIT') + if (queue/'global-stop.json').exists():reason('SEED1_GLOBAL_STOP') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='SEED1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START') + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + try: + if p.exists():value=json.loads(p.read_text()) + except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc))) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(queue) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('fold')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'SEED1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,seed1_started=True,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py new file mode 100644 index 000000000..c1b6966da --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py @@ -0,0 +1,212 @@ +"""Five original grouped folds on released GPUs; train and freeze only, never score.""" +import fcntl +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +import psutil +from phase2_gate_outcome_review import validate_review +from phase2_h1_worker_v2 import checked, save, sha +from phase2_home_freeze_io import freeze +from s2_recovery_gpu_supervisor import disk_usage, tree_usage +from phase3_seed1_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus + + +def run(root): + c, _ = config(root) + validate_review(root) + lock = (root / 'queue.lock').open('a') + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (root / 'started.json').exists(): + raise FileExistsError('One Seed1 attempt; no retry') + for p in psutil.process_iter(['pid', 'cmdline']): + if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []): + raise RuntimeError('Existing Seed1 execution') + os.sched_setaffinity(0, c['coordinator_cpus']) + os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' + start = time.monotonic() + cpu = time.process_time() + active, terminal, estimates, last_epoch = {}, {}, {}, {} + pending = list(c['jobs']) + requested = [] + + def stop(reason): + if not (root / 'global-stop.json').exists(): + save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True)) + + def active_charges(): + return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + for i, v in active.items()} + + def charged(): + return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values()) + + def terminate(v): + if v.get('stop_sent') is None: + if v['process'].poll() is None: + v['process'].send_signal(signal.SIGTERM) + v['stop_sent'] = time.monotonic() + elif time.monotonic() - v['stop_sent'] > 20: + try: + parent = psutil.Process(v['process'].pid) + for p in reversed(parent.children(recursive=True)): + try: + p.kill() + except psutil.NoSuchProcess: + pass + parent.kill() + except psutil.NoSuchProcess: + pass + + def collect(i, v): + rc = v['process'].poll() + if rc is None: + return False + p = root / 'jobs' / str(i) / 'complete.json' + r = read(p) if p.exists() else None + observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + ok = (rc == 0 and r is not None and r['status'] == 'PASS' + and not r.get('monitor_errors') and r.get('worker') is not None + and not r['worker'].get('monitor_errors')) + terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc, + gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0), + guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json')) + v['log'].close() + del active[i] + save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()), + settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()))) + if not ok: + stop('JOB_' + str(i) + '_FAILED') + return True + + def dispatch(): + eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()} + idle,observations=idle_eligible_gpus(eligible) + observed_unix=time.time() + save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations)) + for j in dispatchable(pending, [v['job'] for v in active.values()], idle): + i = j['id'] + d = root / 'jobs' / str(i) + d.mkdir(parents=True, exist_ok=False) + log = (d / 'entry.log').open('x') + begin = time.monotonic() + save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix)) + cmd = [sys.executable, 'tools/phase3_seed1_guard.py', str(root), str(i), sha(d/'assignment.json')] + try: + p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + except BaseException: + log.close() + raise + active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j) + pending.remove(c['jobs'][i]) + save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid'])) + + handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n))) + for s in (signal.SIGTERM, signal.SIGINT)} + save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5, + contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'), + gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False)) + try: + while active or pending: + for i, v in list(active.items()): + collect(i, v) + for reason in requested: + stop(reason) + amount = charged() + _, rss, _ = tree_usage(os.getpid()) + if amount >= c['train_gpu_cap'] - 80: + stop('CUMULATIVE_TRAIN_GPU_BOUNDARY') + if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']: + stop('AGGREGATE_MEMORY_DISK_LIMIT') + if not (root / 'global-stop.json').exists(): + for i in active: + files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json')) + rows = [read(p) for p in files] + if not rows: + continue + epoch = rows[-1]['epoch'] + # Recalculate every epoch: waiting, setup and observed cost stay charged. + if epoch > last_epoch.get(i, 0): + observation = dict(epoch=epoch, charged=active_charges()[i], + max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows), + max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) + cell = project_cell(c['jobs'][i], observation) + estimates[i] = cell + projection = dict(**project_total(c, estimates, terminal, active_charges()), + cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time()) + name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json' + save(root / 'jobs' / str(i) / name, projection) + save(root / 'latest-cost-projection.json', projection) + last_epoch[i] = epoch + if projection['status'] != 'PASS': + stop('PROJECTED_SEED1_BUDGET') + if not (root / 'global-stop.json').exists(): + dispatch() + if (root / 'global-stop.json').exists(): + for v in active.values(): + terminate(v) + save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists() + else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid) + for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending], + charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'])) + if (root / 'global-stop.json').exists() and not active: + break + if active or pending: + time.sleep(1) + if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()): + raise RuntimeError('Seed1 training incomplete') + config(root) + models = [] + for j in c['jobs']: + w = root / 'jobs' / str(j['id']) / 'worker' + r = read(w / 'result.json') + stats = read(checked(w / 'standardization.json', r['standardization_sha256'])) + if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200 + or stats['parts'] != j['learning'] or r['test_targets_read']): + raise ValueError('Terminal model identity') + checked(r['checkpoint'], r['checkpoint_sha256']) + models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'], + standardization=str(w / 'standardization.json'), + standardization_sha256=r['standardization_sha256'])) + if charged() > c['train_gpu_cap']: + raise ValueError('Terminal cumulative cost') + save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False)) + freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json', + *[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]]) + save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()), + training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'], + coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start, + phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(), + no_retry=True, test_targets_read=False, postfit_started=False)) + freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json']) + return 0 + except BaseException as exc: + stop(type(exc).__name__ + ': ' + str(exc)) + while active: + for i, v in list(active.items()): + try: + if not collect(i, v): + terminate(v) + except Exception: + terminate(v) + if v['process'].poll() is not None: + terminal[i] = dict(id=i, status='MALFORMED_TERMINAL', + gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']) + v['log'].close() + del active[i] + if active: + time.sleep(.5) + save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()), + never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(), + postfit_started=False, no_retry=True)) + return 1 + finally: + for s, handler in handlers.items(): + signal.signal(s, handler) + + +if __name__ == '__main__': + raise SystemExit(run(Path(sys.argv[1]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py new file mode 100644 index 000000000..4307eff67 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py @@ -0,0 +1,147 @@ +"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates.""" +import math +import statistics +from pathlib import Path +from phase2_h1_worker_v2 import checked,sha +from phase2_home_freeze_io import verify +from track_a_h2_common import read + +GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'] + + +def execution_command(args): + return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py','phase3_seed1_v2_queue.py','phase3_seed1_v2_guard.py','phase3_seed1_v2_worker.py'} for x in args) + + +def fold_view(pilot,fold): + if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds') + f=pilot['folds'][fold] + if f['fold']!=fold:raise ValueError('Actual original fold identity') + roles=[set(f[k]) for k in ('learning','validation','test')] + if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') + return dict(pilot,folds=[f]) + + +def config(root): + c=read(root/'contract.json');verify(root,'implementation') + if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope') + if c['approval']!='1794990750.013949' or c['accounting_approval']!='1794990899.609759' or c['semantic_approval']!='1794990914.918719' or c['dispatch_approval']!='1794983024.445549':raise PermissionError('Exact v2 rulings') + for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) + old=Path('/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18');prior=old.with_name('phase3-seed1-continuation-2026-11-18') + if root.resolve()!=old.with_name('phase3-seed1-v2-2026-11-18'):raise PermissionError('Sole new labelled v2 root') + if c['continuation_number']!=2 or not c['no_further_continuation'] or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed v2 scope') + costs=[] + for p in [old,prior]: + d=read(p/'complete.json') + if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or sorted(e['id'] for e in d['entries'])!=[0,1]:raise PermissionError('Both immutable old stops') + if sum(e['gpu_seconds_charged'] for e in d['entries'])!=d['training_gpu_seconds']:raise ValueError('Historical ledger mismatch') + costs.append(dict(root=str(p),gpu_seconds=d['training_gpu_seconds'])) + old_spent=sum(d['gpu_seconds'] for d in costs);oc=read(old/'contract.json') + if c['old_attempts']!=costs or c['old_seed1_gpu_once']!=old_spent or c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+old_spent:raise ValueError('Both old charges exactly once') + if c['gpu_cap']!=167200-old_spent or c['train_gpu_cap']!=160000-old_spent or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000 or c['cumulative_seed1_gpu_cap']!=167200:raise ValueError('Unchanged cumulative ceiling') + if c['pending_rule']!='median_current_started_projections_else_train_remaining_div5' or c['fold_stop_rule']!='actual_charged_runaway_only' or c['runaway_gpu_seconds']!=40000:raise ValueError('Frozen total-projection rule') + if any(j['gpu_seconds']!=40000 or j['cpu_seconds']!=40000 for j in c['jobs']):raise ValueError('Actual per-fold runaway guard') + if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged') + pilot=read(c['pilot_contract']) + import numpy as np,torch + if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions') + if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve') + tests=[] + for j in c['jobs']: + f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test']) + for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']: + if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k) + if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved all epochs') + if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition') + return c,pilot + + +def project_cell(job,observation): + e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged'] + validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0)) + if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing') + future=1.25*((200-e)*train+(20-e//10)*validation)+120 + value=charged+future + return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,actual_charged_gpu_seconds=charged,actual_within_runaway=charged<=job['gpu_seconds'],projected_above_runaway=value>job['gpu_seconds']) + + +def project_total(c,estimates,terminal,active_charges=None): + active_charges=active_charges or {};ids={j['id'] for j in c['jobs']} + if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job') + def valid(v): + if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger') + return v + values={} + for i in ids: + if i in terminal:values[i]=valid(terminal[i]['gpu_seconds_charged']) + elif i in estimates:values[i]=max(valid(active_charges.get(i,estimates[i].get('actual_charged_gpu_seconds',0))),valid(estimates[i]['projected_gpu_seconds'])) + # Measured started folds (including finished actuals) determine the median. + # An active fold before its first measured epoch is conservatively reserved at + # the larger of the initial equal share and this current measured median. + initial=c['train_gpu_cap']/len(c['jobs']) + median=statistics.median(values.values()) if values else initial + for i,charged in active_charges.items(): + if i not in values:values[i]=max(valid(charged),initial,median) + pending_reserve=statistics.median(values.values()) if values else initial + rows=[];total=c['postfit_gpu_reserve'];actuals=[] + for j in c['jobs']: + i=j['id'];v=values.get(i,pending_reserve) + actual=valid(terminal[i]['gpu_seconds_charged'] if i in terminal else active_charges.get(i,estimates.get(i,{}).get('actual_charged_gpu_seconds',0))) + total+=v;actuals.append(actual) + rows.append(dict(id=i,projected_gpu_seconds=v,actual_charged_gpu_seconds=actual,actual_within_runaway=actual<=c['runaway_gpu_seconds'],basis='terminal_actual' if i in terminal else 'measured_projection' if i in estimates else 'active_unmeasured_reserve' if i in active_charges else 'pending_median_reserve')) + ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['actual_within_runaway'] for r in rows) + return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve'],pending_reserve_gpu_seconds=pending_reserve,actual_runaway_gpu_seconds=c['runaway_gpu_seconds'],total_projection_within_cap=total<=c['gpu_cap']) + + +def first_epoch_release(root,jobid): + if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop') + p=root/'jobs'/str(jobid)/'first-epoch-projection.json' + if not p.exists():return False + r=read(p) + if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected') + return True + + +def dispatchable(pending,active,available): + occupied={j['gpu_uuid'] for j in active} + if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU') + free=[u for u in GPUS if u in available and u not in occupied] + return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)] + + +def idle_eligible_gpus(eligible,query=None): + if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU') + if query is None: + from s2_precision_continuation_guard import device_usage + query=device_usage + idle=set();observations=[] + for u in GPUS: + if u not in eligible:continue + mem,util=query(u) + if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy') + ok=mem<=128*2**20 and util==0 + observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok)) + if ok:idle.add(u) + return idle,observations + + +def assigned_job(root,c,index,digest): + p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p) + if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity') + return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid']) + + +def available_gpus(c): + import psutil + available={GPUS[0]} + item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root']) + names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []} + if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}: + available.add(GPUS[1]) + if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}: + # GPU2 stays reserved for the already-prioritized Track A postfit until its terminal. + post=Path(c['track_a_postfit_root']) + available.add(GPUS[3]) + if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}: + available.add(GPUS[2]) + return available diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py new file mode 100644 index 000000000..cdf90e8e3 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py @@ -0,0 +1,88 @@ +"""Independent bounded SEED1 fold guard; shared global stop and immutable S1.""" +import fcntl +import json +import os +from pathlib import Path +import signal +import sys +import threading +import time +import psutil +from phase3_seed1_v2_common import config,assigned_job +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage + +def run(queue,fold_id,assignment_sha): + c,_=config(queue);job=assigned_job(queue,c,fold_id,assignment_sha);c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus']) + with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-seed1-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'started.json').exists():raise FileExistsError('Single SEED1 fold launch, no retry') + start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None + save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds'])) + def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start + def reason(x): + if x not in reasons:reasons.append(x) + command=[sys.executable,'tools/phase3_seed1_v2_worker.py',str(queue),str(fold_id),assignment_sha] + def kill_worker(): + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} + def scan(): + if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) + if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') + if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') + if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') + if elapsed()>=c['gpu_seconds']-5:reason('SEED1_FOLD_TIME_LIMIT') + if (queue/'global-stop.json').exists():reason('SEED1_GLOBAL_STOP') + for x in requested:reason(x) + save(root/'live-progress.json',dict(stage='SEED1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) + except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) + if reasons:kill_worker() + done.wait(.5) + thread=None + try: + for i in range(3): + mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) + if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') + if i<2:time.sleep(2) + save(root/'idle-polls.json',samples) + if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START') + thread=threading.Thread(target=watch,daemon=True);thread.start() + remaining=c['gpu_seconds']-elapsed()-10 + receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) + except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() + finally: + done.set() + if thread:thread.join(timeout=4) + if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') + try:scan() + except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) + for s,handler in old.items():signal.signal(s,handler) + value=None;p=root/'worker/result.json' + try: + if p.exists():value=json.loads(p.read_text()) + except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc))) + if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') + if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') + if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') + if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') + totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu + if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') + try:config(queue) + except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) + if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('fold')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT') + save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'SEED1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors, + worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu, + worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,seed1_started=True,no_retry=True)) + return 0 if not reasons else 1 + +if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py new file mode 100644 index 000000000..dd41ca7c8 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py @@ -0,0 +1,34 @@ +"""Epoch-boundary host observations only; no scheduling or causal decisions.""" +import csv +import io +import os +from pathlib import Path +import subprocess +import time +import psutil +from phase2_h1_worker_v2 import save + +def epoch_host_observation(root,fold,epoch,gpu_uuid): + record=dict(unix=time.time(),fold=fold,completed_epoch=epoch,assigned_gpu_uuid=gpu_uuid,observation_only=True,sampling='After the coordinator reads the completed-epoch receipt; not continuous monitoring and not causal proof',gpu_processes=[],errors=[]) + try:record['load_average']=list(os.getloadavg()) + except OSError as exc:record['errors'].append(dict(stage='load_average',type=type(exc).__name__,reason=str(exc))) + own=set() + try: + for p in psutil.process_iter(['pid','cmdline']): + args=p.info['cmdline'] or [] + if str(root) in args and any(Path(a).name=='phase3_seed1_v2_worker.py' for a in args):own.add(p.pid) + except Exception as exc:record['errors'].append(dict(stage='own_worker_identity',type=type(exc).__name__,reason=str(exc))) + try: + result=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=5,check=True) + for row in csv.reader(io.StringIO(result.stdout)): + if not row:continue + if len(row)!=4:raise ValueError('Unexpected GPU process columns') + uuid,pid,name,memory=[s.strip() for s in row];pid=int(pid) + record['gpu_processes'].append(dict(gpu_uuid=uuid,pid=pid,process_name=name,used_memory_mib=memory,own_v2_worker=pid in own,on_assigned_gpu=uuid==gpu_uuid,foreign_on_assigned_gpu=uuid==gpu_uuid and pid not in own)) + except Exception as exc:record['errors'].append(dict(stage='gpu_process_snapshot',type=type(exc).__name__,reason=str(exc))) + record['own_worker_pids']=sorted(own);record['status']='OBSERVED' if not record['errors'] else 'OBSERVATION_INCOMPLETE' + p=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json' + p.parent.mkdir(parents=True,exist_ok=True) + if p.exists():raise FileExistsError('Preserve original epoch observation') + save(p,record) + return record diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py new file mode 100644 index 000000000..7de984fab --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py @@ -0,0 +1,214 @@ +"""Five original grouped folds on released GPUs; train and freeze only, never score.""" +import fcntl +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +import psutil +from phase2_gate_outcome_review import validate_review +from phase2_h1_worker_v2 import checked, save, sha +from phase2_home_freeze_io import freeze +from s2_recovery_gpu_supervisor import disk_usage, tree_usage +from phase3_seed1_v2_observation import epoch_host_observation +from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus + + +def run(root): + c, _ = config(root) + validate_review(root) + lock = (root / 'queue.lock').open('a') + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (root / 'started.json').exists(): + raise FileExistsError('One Seed1 attempt; no retry') + for p in psutil.process_iter(['pid', 'cmdline']): + if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []): + raise RuntimeError('Existing Seed1 execution') + os.sched_setaffinity(0, c['coordinator_cpus']) + os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' + start = time.monotonic() + cpu = time.process_time() + active, terminal, estimates, last_epoch = {}, {}, {}, {} + pending = list(c['jobs']) + requested = [] + + def stop(reason): + if not (root / 'global-stop.json').exists(): + save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True)) + + def active_charges(): + return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + for i, v in active.items()} + + def charged(): + return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values()) + + def terminate(v): + if v.get('stop_sent') is None: + if v['process'].poll() is None: + v['process'].send_signal(signal.SIGTERM) + v['stop_sent'] = time.monotonic() + elif time.monotonic() - v['stop_sent'] > 20: + try: + parent = psutil.Process(v['process'].pid) + for p in reversed(parent.children(recursive=True)): + try: + p.kill() + except psutil.NoSuchProcess: + pass + parent.kill() + except psutil.NoSuchProcess: + pass + + def collect(i, v): + rc = v['process'].poll() + if rc is None: + return False + p = root / 'jobs' / str(i) / 'complete.json' + r = read(p) if p.exists() else None + observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] + ok = (rc == 0 and r is not None and r['status'] == 'PASS' + and not r.get('monitor_errors') and r.get('worker') is not None + and not r['worker'].get('monitor_errors')) + terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc, + gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0), + guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json')) + v['log'].close() + del active[i] + save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()), + settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()))) + if not ok: + stop('JOB_' + str(i) + '_FAILED') + return True + + def dispatch(): + eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()} + idle,observations=idle_eligible_gpus(eligible) + observed_unix=time.time() + save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations)) + for j in dispatchable(pending, [v['job'] for v in active.values()], idle): + i = j['id'] + d = root / 'jobs' / str(i) + d.mkdir(parents=True, exist_ok=False) + log = (d / 'entry.log').open('x') + begin = time.monotonic() + save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix)) + cmd = [sys.executable, 'tools/phase3_seed1_v2_guard.py', str(root), str(i), sha(d/'assignment.json')] + try: + p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + except BaseException: + log.close() + raise + active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j) + pending.remove(c['jobs'][i]) + save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid'])) + + handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n))) + for s in (signal.SIGTERM, signal.SIGINT)} + save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5, + contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'), + gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False)) + try: + while active or pending: + for i, v in list(active.items()): + collect(i, v) + for reason in requested: + stop(reason) + amount = charged() + _, rss, _ = tree_usage(os.getpid()) + if amount >= c['train_gpu_cap'] - 80: + stop('CUMULATIVE_TRAIN_GPU_BOUNDARY') + if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']: + stop('AGGREGATE_MEMORY_DISK_LIMIT') + if not (root / 'global-stop.json').exists(): + for i in active: + files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json')) + rows = [read(p) for p in files] + if not rows: + continue + epoch = rows[-1]['epoch'] + # Recalculate every epoch: waiting, setup and observed cost stay charged. + if epoch > last_epoch.get(i, 0): + epoch_host_observation(root, i, epoch, active[i]['job']['gpu_uuid']) + observation = dict(epoch=epoch, charged=active_charges()[i], + max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows), + max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) + cell = project_cell(c['jobs'][i], observation) + estimates[i] = cell + projection = dict(**project_total(c, estimates, terminal, active_charges()), + cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time()) + name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json' + save(root / 'jobs' / str(i) / name, projection) + save(root / 'latest-cost-projection.json', projection) + last_epoch[i] = epoch + if projection['status'] != 'PASS': + stop('PROJECTED_SEED1_BUDGET') + if not (root / 'global-stop.json').exists(): + dispatch() + if (root / 'global-stop.json').exists(): + for v in active.values(): + terminate(v) + save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists() + else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid) + for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending], + charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'])) + if (root / 'global-stop.json').exists() and not active: + break + if active or pending: + time.sleep(1) + if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()): + raise RuntimeError('Seed1 training incomplete') + config(root) + models = [] + for j in c['jobs']: + w = root / 'jobs' / str(j['id']) / 'worker' + r = read(w / 'result.json') + stats = read(checked(w / 'standardization.json', r['standardization_sha256'])) + if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200 + or stats['parts'] != j['learning'] or r['test_targets_read']): + raise ValueError('Terminal model identity') + checked(r['checkpoint'], r['checkpoint_sha256']) + models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'], + standardization=str(w / 'standardization.json'), + standardization_sha256=r['standardization_sha256'])) + if charged() > c['train_gpu_cap']: + raise ValueError('Terminal cumulative cost') + save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False)) + freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json', + *[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]]) + save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()), + training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'], + coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start, + phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(), + no_retry=True, test_targets_read=False, postfit_started=False)) + freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json']) + return 0 + except BaseException as exc: + stop(type(exc).__name__ + ': ' + str(exc)) + while active: + for i, v in list(active.items()): + try: + if not collect(i, v): + terminate(v) + except Exception: + terminate(v) + if v['process'].poll() is not None: + terminal[i] = dict(id=i, status='MALFORMED_TERMINAL', + gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']) + v['log'].close() + del active[i] + if active: + time.sleep(.5) + save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()), + never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(), + postfit_started=False, no_retry=True)) + return 1 + finally: + for s, handler in handlers.items(): + signal.signal(s, handler) + + +if __name__ == '__main__': + raise SystemExit(run(Path(sys.argv[1]))) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py new file mode 100644 index 000000000..c7767aec1 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py @@ -0,0 +1,67 @@ +"""One new S2 fold, same frozen recipe with seed1/200epochs; no test prediction or target reads.""" +import json,os,sys,time,traceback,random +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import selected_checkpoint +from phase3_seed1_v2_common import config,fold_view,first_epoch_release,assigned_job +from phase2_h2_data import standardization +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root,fold_id,assignment_sha): + c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256'])) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(1);np.random.seed(1);torch.manual_seed(1);torch.cuda.manual_seed_all(1) + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=1,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([1,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch==1: + stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes)) + while not first_epoch_release(root,fold_id):time.sleep(.5) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=1,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py new file mode 100644 index 000000000..6dc641d3a --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py @@ -0,0 +1,67 @@ +"""One new S2 fold, same frozen recipe with seed1/200epochs; no test prediction or target reads.""" +import json,os,sys,time,traceback,random +from pathlib import Path +import numpy as np +import torch +from phase2_h2_s1_common import selected_checkpoint +from phase3_seed1_common import config,fold_view,first_epoch_release,assigned_job +from phase2_h2_data import standardization +from phase2_h2_data import load_part,to_device,arguments,cached +from phase2_h2_model import H2Geo,optimizer,lr_epoch +from phase2_h2_pilot_worker import peaks,train_step +from phase2_residual_math import residual_target +from phase2_h1_worker_v2 import save,sha + +def run(root,fold_id,assignment_sha): + c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() + os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) + start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 + def record(item,epoch_file=False): + item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) + with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') + save(out/'progress.json',item) + if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) + return item + try: + if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') + stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256'])) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(1);np.random.seed(1);torch.manual_seed(1);torch.cuda.manual_seed_all(1) + model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] + save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=1,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) + for epoch in range(1,201): + stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. + if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') + order=np.random.default_rng(np.random.SeedSequence([1,epoch])).permutation(parts) + for index in order: + case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') + result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] + record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) + del data + if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') + metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) + if epoch%10==0: + stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() + with torch.no_grad(): + for case in fold['validation']: + data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) + if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') + with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') + target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') + delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p + torch.cuda.synchronize() + if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') + metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) + stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) + if epoch==1: + stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes)) + while not first_epoch_release(root,fold_id):time.sleep(.5) + if epoch<200: + for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) + stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) + torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=1,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) + save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) + except BaseException as exc: + memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) + save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise +if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py new file mode 100644 index 000000000..1094acb69 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py @@ -0,0 +1,43 @@ +"""Metadata-only seed1 five-fold preparation; immutable seed0 and Track A sources.""" +from pathlib import Path +import time +from phase2_h1_worker_v2 import checked,sha,save +from phase2_home_freeze_io import verify +from phase2_h2_s2_common import config as original_config +from track_a_h2_prepare import local_code_closure +from track_a_h2_common import read + +B=Path(__file__).resolve().parents[1];H=Path('/home/frank14f/MeltMind-runs');R=H/'phase3-seed1-2026-11-18' + +def main(): + start=time.process_time() + if R.exists():raise FileExistsError('Existing seed1 root; no duplicate') + old=H/'phase2-h2-s2-2026-11-17';post=H/'phase2-h2-s2-postfit-2026-11-17';item=H/'phase3-item1-postfit-recovery-2026-11-18' + oc,pilot=original_config(old);verify(post,'terminal');verify(item,'terminal') + pd=read(post/'complete.json');idone=read(item/'complete.json');cost=read(item/'post-terminal-cost.json') + if pd['decision']!='CONFIRMED' or pd['status']!='COMPLETE' or idone['status']!='COMPLETE' or not cost['within_cap']:raise PermissionError('Prior approved stage terminal required') + source=dict(oc['source_sha256']) + for p,h in oc['code_sha256'].items():checked(p,h);source[p]=h + spec=B/'references/2026-11-18/phase3-seed-sensitivity/mold-seed-ensemble-spec.md';checked(spec,'5dce5d0feb2b5cdd2703f53f0030b8127915d3464c7fa37e16507303b9c52b0d') + for p in [old/'contract.json',old/'complete.json',post/'complete.json',post/'post-terminal-cost.json',post/'score/result.json',post/'predictions-manifest.json',post/'contract.json',item/'complete.json',item/'post-terminal-cost.json',item/'terminal-git-freeze.json',item/'terminal-authorization.json',spec]:source[str(p)]=sha(p) + jobs=[];timings=[] + for i,f in enumerate(pilot['folds']): + hist=B/'artifacts/phase2-h2-s1-2026-11-17/worker/history' if i==0 else old/'folds'/str(i)/'worker/history' + records=[hist/f'{e:03d}.json' for e in range(10,201,10)] + for p in records:source[str(p)]=sha(p) + val=max(read(p)['validation_seconds'] for p in records) + jobs.append(dict(id=i,fold=i,learning=f['learning'],validation=f['validation'],test=f['test'],learning_nodes=f['learning_nodes'],validation_nodes=f['validation_nodes'],clock_path=f['clock_path'],clock_sha256=f['clock_sha256'],c=f['c'],target_steps=200*len(f['learning']),learning_parts=len(f['learning']),worker_cpus=[2*i,2*i+1],guard_cpus=[10+i],gpu_seconds=22320,cpu_seconds=22320,setup_gpu_seconds=60,validation_seconds_bound=val,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)) + timings.append(dict(fold=i,validation_seconds_bound=val,source_files=[str(p) for p in records])) + codes=local_code_closure([B/'tools'/n for n in ['phase3_seed1_common.py','phase3_seed1_worker.py','phase3_seed1_guard.py','phase3_seed1_queue.py','prepare_phase3_seed1.py','test_phase3_seed1.py']]) + # Include the original scientific closure, including package files not resolved by local AST imports. + for p,h in oc['code_sha256'].items(): + if p not in codes:codes[p]=h + for p,h in source.items():checked(p,h) + R.mkdir() + comparison=dict(version='seed1-training-comparison-v1',source_spec_sha256=sha(spec),scope='five fresh seed1 grouped folds, original seed0 reused only in later separately reviewed postfit',folds=pilot['folds'],seed=1,epochs=200,selection='last_epoch_only',validation='diagnostic only, original every10epochs',metric='unchanged S2 query readout in later postfit',seed2=False,ensemble=False,original_S2_score_sha256=sha(post/'score/result.json'),original_S2_predictions_sha256=sha(post/'predictions-manifest.json')) + save(R/'comparison-contract.json',comparison) + c=dict(version='phase3-seed1-v1',approval='1794982860.347839',dispatch_approval='1794983024.445549',seed=1,epochs=200,new_fits=5,gpu_cap=118800,train_gpu_cap=111600,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,phase3_gpu_cap=612000,phase3_prior_gpu_once=cost['cumulative_item_gpu_seconds'],item1_recovery_root=str(item),track_a_training_root=str(H/'track-a-h2-2026-11-18'),track_a_postfit_root=str(H/'track-a-postfit-2026-11-18'),pilot_contract=oc['pilot_contract'],jobs=jobs,coordinator_cpus=[14,15],aggregate_rss_bytes=64*2**30,aggregate_disk_bytes=100*2**30,source_sha256=source,code_sha256=codes,comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(R/'comparison-contract.json'),dispatch='next_pending_fold_first_allowed_free_gpu; GPU2 reserved until Track A postfit terminal',no_retry=True,no_cross_card_retry=True,postfit_started=False,stop_after='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING',new_seed0_fits=0,new_seed2_fits=0,ensemble_started=False) + save(R/'contract.json',c);save(R/'preparation-audit.json',dict(status='PREPARED_PENDING_LIGHT_DIFF_REVIEW',code_files=len(codes),contract_sha256=sha(R/'contract.json'),timing_sources=timings,metadata_cpu_seconds=time.process_time()-start,source_pins='PASS',scientific_arrays_read=False,new_model_weights_read=False,test_targets_read=False,new_fits_launched=False,phase3_item1_gpu_once=cost['cumulative_item_gpu_seconds'])) + print(read(R/'preparation-audit.json')|{'timing_sources':'five original per-fold validation timing maxima'}) + +if __name__=='__main__':main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py new file mode 100644 index 000000000..87e7df63e --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py @@ -0,0 +1,43 @@ +"""One labelled continuation: metadata-only, original STOP and scientific sources immutable.""" +import copy +import shutil +import time +from pathlib import Path +from phase2_h1_worker_v2 import checked,sha,save +from phase2_home_freeze_io import verify +from phase3_seed1_common import config as original_config,read +from track_a_h2_prepare import local_code_closure +B=Path(__file__).resolve().parents[1] +OLD=Path('/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18') +ROOT=OLD.with_name('phase3-seed1-continuation-2026-11-18') + +def remaining_contract(original,terminal): + c=copy.deepcopy(original) + spent={e['id']:e['gpu_seconds_charged'] for e in terminal['entries']} + old_charge=terminal['training_gpu_seconds'] + if terminal['status']!='STOPPED' or terminal['never_attempted']!=[2,3,4] or list(spent)!=[0,1] or sum(spent.values())!=old_charge:raise ValueError('Original stopped scope and accounting') + c.update(version='phase3-seed1-labelled-continuation-v1',approval='1794984996.938059',accounting_approval='1794985731.591689',original_seed1_root=str(OLD),original_stop_publication='a45e375d0e8797e4d63258e446647191ef253585',continuation_number=1,fresh_restart_folds=[0,1],previously_unattempted_folds=[2,3,4],old_seed1_gpu_once=old_charge,cumulative_seed1_gpu_cap=167200,cumulative_fold_gpu_cap=32000,gpu_cap=167200-old_charge,train_gpu_cap=160000-old_charge,phase3_prior_gpu_once=original['phase3_prior_gpu_once']+old_charge,accounting='Runtime counters are new-run only. Add old_seed1_gpu_once once for cumulative seed1; phase3_prior already includes it. Never add it again to phase3. Remaining per-fold caps debit original fold charges; old STOP remains immutable.',remaining_caps_final=True) + for j in c['jobs']: + j['gpu_seconds']=32000-spent.get(j['id'],0);j['cpu_seconds']=j['gpu_seconds'] + return c + +def main(): + start=time.process_time() + if ROOT.exists():raise FileExistsError('Existing sole continuation root') + oc,_=original_config(OLD);proof=verify(OLD,'budget-stop') + if proof['commit']!='a556b58340410dbe3faeb2dc2143b3708a7e724b':raise ValueError('Original STOP freeze') + c=remaining_contract(oc,read(OLD/'complete.json')) + source=dict(oc['source_sha256']);source.update(oc['code_sha256']);source.update(proof['files']) + for p in [OLD/'budget-stop-git-freeze.json',OLD/'budget-stop-authorization.json',OLD/'expected-review-binding.json']: + source[str(p)]=sha(p) + files=[B/'tools'/n for n in ['phase3_seed1_continuation_common.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py','prepare_phase3_seed1_continuation.py','test_phase3_seed1_continuation.py']] + codes=local_code_closure(files) + for p,h in oc['code_sha256'].items():codes.setdefault(p,h) + for p,h in source.items():checked(p,h) + ROOT.mkdir() + shutil.copyfile(OLD/'comparison-contract.json',ROOT/'comparison-contract.json') + c.update(source_sha256=source,code_sha256=codes,comparison_contract=str(ROOT/'comparison-contract.json')) + save(ROOT/'contract.json',c) + save(ROOT/'preparation-audit.json',dict(status='PREPARED_PENDING_NARROW_REVIEW',contract_sha256=sha(ROOT/'contract.json'),code_files=len(codes),source_files=len(source),source_pins='PASS',old_stop_freeze=proof['commit'],old_seed1_gpu_once=c['old_seed1_gpu_once'],new_train_gpu_cap=c['train_gpu_cap'],new_gpu_cap=c['gpu_cap'],cumulative_seed1_gpu_cap=c['cumulative_seed1_gpu_cap'],new_fold_caps=[j['gpu_seconds'] for j in c['jobs']],phase3_prior_gpu_once=c['phase3_prior_gpu_once'],metadata_cpu_seconds=time.process_time()-start,scientific_arrays_read=False,model_weights_read=False,test_targets_read=False,new_fits_launched=False,changes='Remaining budget/contract accounting; isolated entry names; original science, projection, idle/first-epoch gates unchanged')) + print(read(ROOT/'preparation-audit.json')) +if __name__=='__main__':main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py new file mode 100644 index 000000000..c0a4ff3d1 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py @@ -0,0 +1,47 @@ +"""Metadata-only v2 binding; debit both terminal attempts once, no training.""" +import copy +import shutil +import time +from pathlib import Path +from phase2_h1_worker_v2 import checked,sha,save +from phase2_home_freeze_io import verify +from phase3_seed1_common import config as original_config,read +from phase3_seed1_continuation_common import config as continuation_config +from track_a_h2_prepare import local_code_closure +B=Path(__file__).resolve().parents[1] +OLD=Path('/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18') +PRIOR=OLD.with_name('phase3-seed1-continuation-2026-11-18') +ROOT=OLD.with_name('phase3-seed1-v2-2026-11-18') + +def remaining_contract(original,attempts): + c=copy.deepcopy(original);costs=[] + for root,terminal in attempts: + if terminal['status']!='STOPPED' or terminal['never_attempted']!=[2,3,4] or sorted(e['id'] for e in terminal['entries'])!=[0,1]:raise ValueError('Both stopped scopes') + if sum(e['gpu_seconds_charged'] for e in terminal['entries'])!=terminal['training_gpu_seconds']:raise ValueError('Exact prior ledger') + costs.append(dict(root=str(root),gpu_seconds=terminal['training_gpu_seconds'])) + old_charge=sum(x['gpu_seconds'] for x in costs) + c.update(version='phase3-seed1-v2-total-projection',approval='1794990750.013949',accounting_approval='1794990899.609759',semantic_approval='1794990914.918719',original_seed1_root=str(OLD),previous_continuation_root=str(PRIOR),previous_stop_publication='d3c94a00694c2e384efd3e002ca37d300866340e',continuation_number=2,no_further_continuation=True,fresh_restart_folds=[0,1],previously_unattempted_folds=[2,3,4],old_attempts=costs,old_seed1_gpu_once=old_charge,cumulative_seed1_gpu_cap=167200,gpu_cap=167200-old_charge,train_gpu_cap=160000-old_charge,phase3_prior_gpu_once=original['phase3_prior_gpu_once']+old_charge,pending_rule='median_current_started_projections_else_train_remaining_div5',active_unmeasured_rule='max(actual_charge,initial_equal_share,current_measured_median); then included in started median',fold_stop_rule='actual_charged_runaway_only',runaway_gpu_seconds=40000,epoch_foreign_gpu_observation='bounded observation-only snapshot after each completed epoch; errors retained, no causal or scheduling conclusion',accounting='Runtime counters new-v2 only; cumulative seed1 adds old_attempts sum once; phase3 prior already contains both old attempts. Per-fold40000 is actual runaway, not partitioned reserve.',remaining_caps_final=True) + for j in c['jobs']:j['gpu_seconds']=40000;j['cpu_seconds']=40000 + return c + +def main(): + start=time.process_time() + if ROOT.exists():raise FileExistsError('Existing sole v2 root') + oc,_=original_config(OLD);continuation_config(PRIOR) + proofs=[verify(OLD,'budget-stop'),verify(PRIOR,'budget-stop')] + if [p['commit'] for p in proofs]!=['a556b58340410dbe3faeb2dc2143b3708a7e724b','e3f251489d273037f47300ac295c174237551b7b']:raise ValueError('Both immutable STOP freezes') + c=remaining_contract(oc,[(p,read(p/'complete.json')) for p in [OLD,PRIOR]]) + source=dict(oc['source_sha256']);source.update(oc['code_sha256']) + for root,proof in zip([OLD,PRIOR],proofs): + source.update(proof['files']) + for p in [root/'budget-stop-git-freeze.json',root/'budget-stop-authorization.json',root/'expected-review-binding.json']:source[str(p)]=sha(p) + files=[B/'tools'/n for n in ['phase3_seed1_v2_common.py','phase3_seed1_v2_observation.py','phase3_seed1_v2_queue.py','phase3_seed1_v2_guard.py','phase3_seed1_v2_worker.py','prepare_phase3_seed1_v2.py','test_phase3_seed1_v2.py']] + codes=local_code_closure(files) + for p,h in read(PRIOR/'contract.json')['code_sha256'].items():codes.setdefault(p,h) + for p,h in source.items():checked(p,h) + ROOT.mkdir();shutil.copyfile(OLD/'comparison-contract.json',ROOT/'comparison-contract.json') + c.update(source_sha256=source,code_sha256=codes,comparison_contract=str(ROOT/'comparison-contract.json')) + save(ROOT/'contract.json',c) + save(ROOT/'preparation-audit.json',dict(status='PREPARED_PENDING_LOGIC_REVIEW',contract_sha256=sha(ROOT/'contract.json'),code_files=len(codes),source_files=len(source),source_pins='PASS',old_stop_freezes=[p['commit'] for p in proofs],old_attempts=c['old_attempts'],old_seed1_gpu_once=c['old_seed1_gpu_once'],new_train_gpu_cap=c['train_gpu_cap'],new_gpu_cap=c['gpu_cap'],cumulative_seed1_gpu_cap=167200,runaway_gpu_seconds=40000,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],initial_pending_reserve=c['train_gpu_cap']/5,metadata_cpu_seconds=time.process_time()-start,scientific_arrays_read=False,model_weights_read=False,test_targets_read=False,new_fits_launched=False)) + print(read(ROOT/'preparation-audit.json')) +if __name__=='__main__':main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py new file mode 100644 index 000000000..5e9422584 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py @@ -0,0 +1,53 @@ +"""Bounded, recorded idle observations; never signal foreign processes.""" +import csv +import json +import subprocess +import time +from pathlib import Path +import psutil + + +def inventory(): + r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3) + return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())} + + +def snapshot(uuid): + r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) + rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1 + memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100 + r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) + occupants=[] + for row in csv.reader(r.stdout.splitlines()): + if not row or row[0].strip()!=uuid:continue + pid=int(row[1]);user=None + try:user=psutil.Process(pid).username() + except (psutil.NoSuchProcess,psutil.AccessDenied):pass + occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip())) + return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants, + idle=memory<=128 and util==0 and not occupants) + + +def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep): + begin=clock();consecutive=0;count=0;last=None + while True: + try:last=sample(uuid) + except Exception as exc: + last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[]) + count+=1;consecutive=consecutive+1 if last['idle'] else 0 + record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin)) + if clock()-begin>limit_seconds:break + if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) + remaining=limit_seconds-(clock()-begin) + if remaining<=0:break + sleep(min(10,remaining)) + return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) + + +def recorded_wait(root,name,uuid,limit_seconds): + path=Path(root)/'idle-gates';path.mkdir(exist_ok=True) + def record(row): + with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n') + result=wait_idle(uuid,limit_seconds,record) + with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n') + return result diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py new file mode 100644 index 000000000..d6be24cf5 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py @@ -0,0 +1,133 @@ +"""Independent future-run guard; no matrix launcher or training entry point. + +Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA +peaks while the worker is alive and again after any exit, including failure. +""" +import hashlib +import json +import math +import os +from pathlib import Path +import threading +import time +import psutil + +from s2_precision_continuation_guard import device_usage +from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage + +ALLOCATOR_CONF='expandable_segments:True' +VRAM_BYTES=15*2**30 +HOST_BYTES=16*2**30 +DISK_BYTES=5*2**30 +PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes') + + +def peaks_in(value): + found={} + if isinstance(value,dict): + for key,child in value.items(): + if key in PEAK_KEYS: + if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0: + raise ValueError('Invalid recorded CUDA peak: '+key) + found[key]=child + nested=peaks_in(child) + for k,v in nested.items():found[k]=max(found.get(k,0),v) + elif isinstance(value,list): + for child in value: + for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v) + return found + + +class PeakRecords: + def __init__(self): + self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[] + def scan(self,root): + # Includes progress, result and failure records, plus epoch/history JSON. + # Running maxima survive file replacement, regression and deletion. + for path in sorted(Path(root).rglob('*.json')): + try: + stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino) + if self.cache.get(str(path))==version:continue + raw=path.read_bytes() + except FileNotFoundError:continue + peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version + if not peaks:continue + record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks) + self.observations.append(record) + for key,value in peaks.items(): + if key not in self.witnesses or value>self.maxima[key]: + self.maxima[key]=value;self.witnesses[key]=record + return max(self.maxima.values())>VRAM_BYTES + def as_dict(self): + return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations, + limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES) + + +def active_command_pids(command): + result=[] + try:children=psutil.Process(os.getpid()).children(recursive=True) + except psutil.NoSuchProcess:return result + for p in children: + try: + if p.cmdline()==command:result.append(p.pid) + except (psutil.NoSuchProcess,psutil.AccessDenied):pass + return result + + +def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds): + root=Path(root) + if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF: + raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn') + assert 00 + begin=time.monotonic();initial,util=device_usage(uuid) + if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN') + tracker=PeakRecords();done=threading.Event() + monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0) + def reason(value): + if value not in monitor['reasons']:monitor['reasons'].append(value) + def scan_records(): + try: + if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT') + except Exception as exc: + reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc)) + def watch(): + while not done.is_set(): + scan_records() + try: + used,_=device_usage(uuid);monitor['samples']+=1 + monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) + if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT') + _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) + if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT') + if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT') + except Exception as exc: + reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc)) + if monitor['reasons']: + for pid in active_command_pids(command):stop_group(pid) + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + remaining=wall_limit-(time.monotonic()-begin)-2.5 + if remaining<=0:raise RuntimeError('No wall budget left before spawn') + receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds, + job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES, + disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid) + finally: + done.set();thread.join(timeout=3) + if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP') + else:scan_records() # Mandatory even if the worker failed or wrote no result. + result_path=root/name/'result.json';result=None + if result_path.exists(): + try:result=json.loads(result_path.read_text()) + except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc)) + if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT') + if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS') + elapsed=time.monotonic()-begin + if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT') + receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason') + if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) + receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed, + allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid, + result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None) + save(root/(name+'-guard-receipt.json'),receipt) + return receipt,result diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py new file mode 100644 index 000000000..f8b08bd21 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py @@ -0,0 +1,119 @@ +"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders. + +No optimiser, training loop, data reader or GPU invocation is provided. +""" +import torch +from torch import nn + +from .blocks import MLP, Transolver_block +from .features import BASE_COLUMNS +from .normalization import BaseStandardizer + + +TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5, + schedule="cosine", batch_parts=1, max_epochs=200, + patience=20, seeds=[0, 1, 2], widths=[128, 256]) + + +def check_tensor(tensor, shape, name): + if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all(): + raise ValueError(f"invalid {name}") + + +class WholePartEncoder(nn.Module): + def __init__(self, width, standardizer): + super().__init__() + if width not in (128, 256): + raise ValueError("S2 permits widths 128 or 256 only") + if not isinstance(standardizer, BaseStandardizer): + raise ValueError("frozen learning-pool standardizer required") + self.standardizer = standardizer + self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64)) + self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64)) + self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False) + self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width, + dropout=0., mlp_ratio=1, slice_num=32, + last_layer=False) for _ in range(4)]) + self.apply(self._initialize) + self.placeholder = nn.Parameter(torch.rand(width)/width) + + @staticmethod + def _initialize(module): + # torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency. + if isinstance(module, nn.Linear): + nn.init.trunc_normal_(module.weight, std=.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + nn.init.zeros_(module.bias) + nn.init.ones_(module.weight) + + def forward(self, base): + if base.ndim != 2 or base.shape[0] == 0: + raise ValueError("one whole part with all tet nodes required") + check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs") + # Keep normalisation in FP32 even under future mixed-precision training. + base = (base.float()-self.base_mean.float()) / self.base_scale.float() + if not torch.isfinite(base).all(): + raise ValueError("nonfinite standardized base input") + fx = self.preprocess(base[None]) + self.placeholder[None, None] + for block in self.blocks: + fx = block(fx) + if not torch.isfinite(fx).all(): + raise ValueError("nonfinite encoder output") + return fx[0] + + +class QueryModel(nn.Module): + def __init__(self, arm, width=128, *, standardizer): + super().__init__() + if arm not in ("N", "E"): + raise ValueError("arm must be N or E") + self.arm, self.width = arm, width + self.encoder = WholePartEncoder(width, standardizer) + query_dim = 2 if arm == "N" else 6 + self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(), + nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1)) + self.decoder.apply(WholePartEncoder._initialize) + + def forward(self, base, query_ids, gaps, edge_q=None): + if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0: + raise ValueError("nonempty int64 original-node query IDs required") + if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base): + raise ValueError("query node outside whole part") + check_tensor(gaps, (len(query_ids), 2), "g12/g23") + if self.arm == "E": + if edge_q is None: + raise ValueError("E requires all four edge summaries") + check_tensor(edge_q, (len(query_ids), 4), "edge summaries") + local = torch.cat([gaps, edge_q], dim=-1) + else: + if edge_q is not None: + raise ValueError("N must not receive edge summaries") + local = gaps + embedding = self.encoder(base)[query_ids] + y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1) + if not torch.isfinite(y).all(): + raise ValueError("nonfinite decoder output") + return y + + def parameter_counts(self): + return {"encoder": sum(p.numel() for p in self.encoder.parameters()), + "decoder": sum(p.numel() for p in self.decoder.parameters()), + "total": sum(p.numel() for p in self.parameters())} + + +def weighted_mse(prediction, target, global_weights): + """Sum of contributions with already globally-normalised hierarchy weights. + + Do not renormalise a single part's weights: that would erase component balance. + """ + if prediction.ndim != 1 or prediction.numel() == 0: + raise ValueError("nonempty query predictions required") + check_tensor(prediction, prediction.shape, "prediction") + check_tensor(target, prediction.shape, "target") + check_tensor(global_weights, prediction.shape, "weights") + if (global_weights <= 0).any(): + raise ValueError("positive hierarchy weights required") + # FP32 arithmetic under future AMP; do not form residuals in float16. + return torch.sum(global_weights.float() * (prediction.float()-target.float()).square()) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py new file mode 100644 index 000000000..447ff1197 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py @@ -0,0 +1,79 @@ +"""Bounded pilot training primitives; corpus runs require a frozen ready manifest.""" +import math +from pathlib import Path +import numpy as np +import torch +from s2_models.network import weighted_mse + + +def part_weights(counts, parts_in_component, component_count, learning_parts): + c=np.asarray(counts,dtype=np.float64) + if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts') + if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy') + # Uniform shuffled part steps estimate the full component-balanced objective. + return c/c.sum()/parts_in_component/component_count*learning_parts + + +def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False,step_info=None): + optimizer.zero_grad(set_to_none=True) + with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp): + prediction=model(base,ids,gaps,edge if model.arm=='E' else None) + loss=weighted_mse(prediction,target,weights) + if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss') + scale_before=float(scaler.get_scale()) if scaler is not None else None + if scaler is None: + loss.backward() + else: + scaler.scale(loss).backward();scaler.unscale_(optimizer) + if scaler is None and any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()): + raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback') + if scaler is None:optimizer.step() + else:scaler.step(optimizer);scaler.update() + scale_after=float(scaler.get_scale()) if scaler is not None else None + skipped=scaler is not None and scale_after20:raise FloatingPointError('More than 20 consecutive AMP skipped steps') + def summary(self): + return dict(attempted_steps=self.attempted,skipped_steps=self.skipped,successful_updates=self.attempted-self.skipped, + skipped_fraction=self.skipped/self.attempted if self.attempted else 0.,consecutive_skipped=self.consecutive) + def finish_epoch(self,epoch): + if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0') + + +def common_epoch_limit(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes): + if min(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes)<=0:raise ValueError('Positive per-node timing and node counts required') + epoch=training_seconds_per_node*learning_nodes+inference_seconds_per_node*validation_nodes + if not math.isfinite(epoch):raise ValueError('Nonfinite epoch time') + limit=min(200,math.floor(25*60/epoch)) + if limit<1:raise ValueError('One projected epoch exceeds the 25-minute run reserve') + return dict(E_max=limit,cosine_T_max=limit,projected_epoch_seconds=epoch,projected_epochs_seconds=limit*epoch, + training_seconds_per_node=training_seconds_per_node,inference_seconds_per_node=inference_seconds_per_node, + learning_nodes=learning_nodes,validation_nodes=validation_nodes) + + +def component_score(part_errors,component_ids): + if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster') + groups={} + for part,value in part_errors.items(): + if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score') + groups.setdefault(component_ids[part],[]).append(value) + return float(np.mean([np.mean(values) for values in groups.values()])) + + +def select_capacity(scores): + # Exactly four run scores, no choice from an incomplete/failed run matrix. + if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required') + if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite') + return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w)) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py new file mode 100644 index 000000000..80961aef1 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py @@ -0,0 +1,122 @@ +"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import psutil +from freeze_g1_components import sha +from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage +from s2_pilot_train import select_capacity +from s2_gpu_idle_gate import recorded_wait + + +def parse_device_usage(text): + lines=text.strip().splitlines() + if len(lines)!=1:raise ValueError('Expected exactly one GPU') + fields=lines[0].split(',') + if len(fields)!=2:raise ValueError('Malformed GPU usage row') + memory,util=map(lambda s:int(s.strip()),fields) + if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters') + return memory*2**20,util + + +def device_usage(uuid): + r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2) + return parse_device_usage(r.stdout) + + +def active_training_pids(parent,root,index): + ids=[] + try:children=psutil.Process(parent).children(recursive=True) + except psutil.NoSuchProcess:return ids + for p in children: + try: + cmd=p.cmdline() + if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid) + except psutil.NoSuchProcess:pass + return ids + + +def guarded_worker(root,spec,i,uuid,wall_limit,command): + run_start=time.monotonic();initial,util=device_usage(uuid) + if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN') + done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0) + def watch(): + previous=time.monotonic() + while not done.is_set(): + now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now + try: + used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) + if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT' + _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) + if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT' + if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT' + except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc) + if monitor['reason']: + for pid in active_training_pids(os.getpid(),root,i):stop_group(pid) + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + wall=wall_limit-(time.monotonic()-run_start)-2.5 + if wall<=0:raise RuntimeError('No budget left before spawn') + r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall, + tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid) + finally: + done.set();thread.join(timeout=3) + total_wall=time.monotonic()-run_start + result_path=root/spec['name']/'result.json' + result=json.loads(result_path.read_text()) if result_path.exists() else None + if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason']) + if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT') + if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT') + r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall, + result_sha256=sha(result_path) if result else None) + save(root/(spec['name']+'-guard-receipt.json'),r) + return r,result + + +def run(root,prior_wall_seconds=0.): + with (root/'pilot.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return run_locked(root,prior_wall_seconds) + + +def run_locked(root,prior_wall_seconds=0.): + c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid'] + prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.)) + assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)] + for p,h in c['code_sha256'].items():assert sha(p)==h,p + assert not (root/'pilot-started.json').exists(),'No automatic pilot retry' + start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) + rows=[];scores={};reason=None + for i,spec in enumerate(c['runs']): + remaining=7200-prior_wall_seconds-(time.monotonic()-start) + if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break + # Initial readiness was proved by the continuation entry; subsequent + # contexts also need a settling window. All waits consume the same budget. + if i>0: + gate=recorded_wait(root,spec['name'],uuid,min(600,remaining-3)) + if gate['status']!='IDLE_CONFIRMED':reason='GPU_IDLE_TIMEOUT_BEFORE_RUN';break + remaining=7200-prior_wall_seconds-(time.monotonic()-start) + if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break + command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)] + try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command) + except Exception as exc:reason=str(exc);break + rows.append(dict(run=spec,**r)) + if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break + scores[(spec['arm'],spec['width'])]=result['best_validation_score'] + if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT' + selected=select_capacity(scores) if len(scores)==4 and reason is None else None + summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected, + selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required', + pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0) + save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py new file mode 100644 index 000000000..6e6930171 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py @@ -0,0 +1,158 @@ +"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', + MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or '')) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py new file mode 100644 index 000000000..b1c7ea515 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py @@ -0,0 +1,40 @@ +"""Race-tolerant sampled infrastructure monitoring, without worker changes.""" +import json +from pathlib import Path +import stat +import sys +import time + + +def disk_usage(root): + logical=allocated=0 + for p in Path(root).rglob('*'): + try:s=p.stat(follow_symlinks=False) + except FileNotFoundError:continue # atomic progress replacement / vanished entry + if stat.S_ISREG(s.st_mode): + logical+=s.st_size;allocated+=s.st_blocks*512 + return max(logical,allocated) + + +class Monitor: + def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None + def record(self,operation,exc): + self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc)) + try: + with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n') + except Exception as log_exc: + print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True) + def sample(self,operation,fn,previous): + try:return fn(),True + except Exception as exc:self.record(operation,exc);return previous,False + def retry(self,operation,fn,poll): + while True: + value,ok=self.sample(operation,fn,None) + if ok:return value + time.sleep(poll) + + +def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30): + if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT' + if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT' + return None diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py new file mode 100644 index 000000000..674a3d1e5 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py @@ -0,0 +1,97 @@ +"""S2 CPU synthetic boundaries; no real learning/validation/test arrays or GPU.""" +import ast,copy,inspect,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +import numpy as np +from phase2_h2_s2_common import fold_view,projection,first_epoch_release,confirmed,execution_command +from phase2_h2_data import load_part,standardization +import phase2_h2_s1_worker as oldworker +import phase2_h2_s2_worker as newworker +import phase2_h2_s2_guard as guard +from phase2_h1_worker_v2 import save,sha +from s2_main_memory_guard import PeakRecords + +class Roles(unittest.TestCase): + def setUp(self): + self.pilot=dict(folds=[dict(fold=i,learning=['learn'+str(i)],validation=['val'+str(i)],test=['test'+str(i)],learning_nodes=2) for i in range(5)],parts={'learn'+str(i):dict(nodes=2) for i in range(5)}) + def test_all_new_folds_map_exact_original_roles_without_mutation(self): + before=copy.deepcopy(self.pilot) + for f in range(1,5): + v=fold_view(self.pilot,f);self.assertEqual(v['folds'][0],self.pilot['folds'][f]);self.assertEqual(v['folds'][0]['fold'],f) + self.assertEqual(self.pilot,before) + def test_fold0_and_role_overlap_rejected(self): + for f in [0,5,-1]: + with self.assertRaises(PermissionError):fold_view(self.pilot,f) + self.pilot['folds'][2]['test']=['learn2'] + with self.assertRaises(ValueError):fold_view(self.pilot,2) + def test_test_members_rejected_before_geometry_or_target_io(self): + for f in range(1,5): + for role in ['learning','validation']: + with self.assertRaises(PermissionError):load_part(fold_view(self.pilot,f),'test'+str(f),{},role,Path('/unused')) + def test_normalization_uses_only_actual_learning(self): + for f in range(1,5): + seen=[] + def features(c,case,name): + self.assertEqual(case,'learn'+str(f));self.assertEqual(name,'features');seen.append(case);return np.vstack([np.zeros(19),np.full(19,2.)]) + with tempfile.TemporaryDirectory() as p,patch('phase2_h2_data.cached',features): + v=standardization(fold_view(self.pilot,f),Path(p));self.assertEqual(v['parts'],['learn'+str(f)]);self.assertEqual(v['mean'],[1.]*19);self.assertEqual(v['scale'],[1.]*19);self.assertEqual(seen,['learn'+str(f)]) + def test_scientific_epoch_loop_AST_unchanged_except_cost_wait(self): + def loop(fn): + t=ast.parse(inspect.getsource(fn));return next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch') + old=loop(oldworker.run);new=loop(newworker.run);new.body=[n for n in new.body if not(isinstance(n,ast.If) and ast.unparse(n.test)=='epoch == 1')] + self.assertEqual(ast.dump(old),ast.dump(new)) + +class BudgetAndGate(unittest.TestCase): + def setUp(self): + self.c=dict(prior_gpu_seconds=18858.21490871499,total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,jobs=[dict(fold=f,gpu_seconds=(144000-18858.21490871499-2400)/4,validation_seconds_bound=12.) for f in range(1,5)]) + self.obs={f:dict(epoch=1,charged_gpu_seconds=200.,max_train_epoch_seconds=90.,max_validation_seconds=0.) for f in range(1,5)} + def test_old_cost_once_plus_all_four_waits_and_prediction_reserve(self): + d=projection(self.c,self.obs);expected=18858.21490871499+4*(200+1.25*(199*90+20*12)+60)+2400;self.assertEqual(d['projected_total_gpu_seconds'],expected);self.assertEqual(d['status'],'PASS') + def test_budget_and_per_job_limits_stop(self): + o=copy.deepcopy(self.obs);o[1]['max_train_epoch_seconds']=150;self.assertEqual(projection(self.c,o)['status'],'STOP_BUDGET_PROJECTION') + for v in o.values():v['max_train_epoch_seconds']=200 + self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],144000) + def test_missing_epoch_invalid_timing_rejected(self): + with self.assertRaises(ValueError):projection(self.c,{1:self.obs[1]}) + for field,value in [('epoch',0),('max_train_epoch_seconds',float('nan')),('charged_gpu_seconds',-1)]: + o=copy.deepcopy(self.obs);o[2][field]=value + with self.assertRaises(ValueError):projection(self.c,o) + def test_actual_validation_cost_can_raise_projection(self): + a=projection(self.c,self.obs);o=copy.deepcopy(self.obs);o[1]['max_validation_seconds']=30;self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],a['projected_total_gpu_seconds']) + def test_epoch10_and_final_remaining_validation_count(self): + o=copy.deepcopy(self.obs) + for v in o.values():v['epoch']=200 + self.assertAlmostEqual(projection(self.c,o)['projected_total_gpu_seconds'],18858.21490871499+4*(200+60)+2400) + def test_first_epoch_barrier_requires_exact_pass(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);self.assertFalse(first_epoch_release(p,'fixed')) + for result in [dict(status='STOP_BUDGET_PROJECTION',contract_sha256='fixed'),dict(status='PASS',contract_sha256='other')]: + (p/'first-epoch-projection.json').write_text(json.dumps(result)) + with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed') + (p/'first-epoch-projection.json').write_text(json.dumps(dict(status='PASS',contract_sha256='fixed')));self.assertTrue(first_epoch_release(p,'fixed')) + (p/'global-stop.json').write_text('{}') + with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed') + def test_confirmation_is_strict_and_complete_five_fold_only(self): + self.assertTrue(confirmed(-.001,552,204)) + for h,n,g in [(0,552,204),(.001,552,204),(None,552,204),(float('nan'),552,204),(-.001,121,37),(-.001,551,204),(-.001,552,203)]:self.assertFalse(confirmed(h,n,g)) + def test_exact_process_identity_excludes_preparation(self): + self.assertFalse(execution_command(['python','tools/prepare_phase2_h2_s2.py']));self.assertTrue(execution_command(['python','tools/phase2_h2_s2_queue.py'])) + def test_failure_peak_remains_even_after_lower_final_peak(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);t=PeakRecords();(p/'failure.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=16*2**30)));self.assertTrue(t.scan(p));(p/'failure.json').unlink();(p/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=200)));self.assertTrue(t.scan(p));self.assertEqual(t.maxima['peak_reserved_bytes'],16*2**30) +class GuardLifecycle(unittest.TestCase): + def exercise(self,peak): + with tempfile.TemporaryDirectory() as d: + q=Path(d);save(q/'contract.json',dict(synthetic=True));r=q/'folds/1';r.mkdir(parents=True) + job=dict(fold=1,gpu_uuid='synthetic',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30) + calls=[] + def worker(command,*args,**kwargs): + calls.append(command);self.assertEqual(command[-2:],[str(q),'1']);w=r/'worker';w.mkdir();save(w/'result.json',dict(status='COMPLETE',fold=1,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=[]) + with patch.object(guard,'config',return_value=(dict(jobs=[job]),{})),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',worker),patch.object(guard.time,'sleep'): + rc=guard.run(q,1) + self.assertEqual(len(calls),1);self.assertEqual(json.loads((r/'started.json').read_text())['contract_sha256'],sha(q/'contract.json'));return rc,json.loads((r/'complete.json').read_text()) + def test_queue_contract_identity_and_complete_guard(self): + rc,d=self.exercise(100);self.assertEqual(rc,0);self.assertEqual(d['status'],'PASS') + def test_recorded_failure_peak_cannot_be_ignored(self): + rc,d=self.exercise(16*2**30);self.assertEqual(rc,1);self.assertIn('RECORDED_VRAM_LIMIT',d['reasons']) +if __name__=='__main__':unittest.main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py new file mode 100644 index 000000000..5e2531e3b --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py @@ -0,0 +1,105 @@ +"""CPU synthetic seed/role/cost/release/guard tests; no real arrays or GPU.""" +import ast,copy,inspect,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +import numpy as np +from phase3_seed1_common import fold_view,project_cell,project_total,dispatchable,first_epoch_release,available_gpus,GPUS,assigned_job,idle_eligible_gpus +from phase2_h2_data import load_part,standardization +from phase2_h1_worker_v2 import save,sha +import phase2_h2_s2_worker as original +import phase3_seed1_worker as new +import phase3_seed1_guard as guard + +class SeedTests(unittest.TestCase): + def setUp(self): + self.pilot=dict(folds=[dict(fold=i,learning=['l'+str(i)],validation=['v'+str(i)],test=['t'+str(i)],learning_nodes=2) for i in range(5)],parts={'l'+str(i):dict(nodes=2) for i in range(5)}) + self.jobs=[dict(id=i,gpu_seconds=22320,validation_seconds_bound=20) for i in range(5)] + self.c=dict(jobs=self.jobs,gpu_cap=118800,postfit_gpu_reserve=7200,phase3_gpu_cap=612000,phase3_prior_gpu_once=73000) + def test_all_five_original_roles(self): + old=copy.deepcopy(self.pilot) + for i in range(5):self.assertEqual(fold_view(self.pilot,i)['folds'][0],self.pilot['folds'][i]) + self.assertEqual(old,self.pilot) + def test_foreign_and_overlap_rejected(self): + for i in [True,-1,5]: + with self.assertRaises(PermissionError):fold_view(self.pilot,i) + self.pilot['folds'][0]['test']=['l0'] + with self.assertRaises(ValueError):fold_view(self.pilot,0) + def test_test_role_never_used_for_train_or_normalization(self): + for i in range(5): + v=fold_view(self.pilot,i) + for role in ['learning','validation']: + with self.assertRaises(PermissionError):load_part(v,'t'+str(i),{},role,Path('/unused')) + def features(c,case,name): + self.assertEqual((case,name),('l'+str(i),'features'));return np.vstack([np.zeros(19),np.full(19,2.)]) + with tempfile.TemporaryDirectory() as d,patch('phase2_h2_data.cached',features): + s=standardization(v,Path(d));self.assertEqual(s['parts'],['l'+str(i)]);self.assertEqual(s['mean'],[1.]*19) + def test_epoch_math_identical_except_seed_and_cost_wait(self): + def loop(fn): + t=ast.parse(inspect.getsource(fn));n=next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch') + n.body=[b for b in n.body if not(isinstance(b,ast.If) and ast.unparse(b.test)=='epoch == 1')] + return n + a,b=loop(original.run),loop(new.run) + seq=[n for n in ast.walk(b) if isinstance(n,ast.Call) and ast.unparse(n.func)=='np.random.SeedSequence'];self.assertEqual(len(seq),1);self.assertEqual(seq[0].args[0].elts[0].value,1);seq[0].args[0].elts[0].value=0 + self.assertEqual(ast.dump(a),ast.dump(b)) + def test_all_rng_seed1_and_final_checkpoint(self): + tree=ast.parse(inspect.getsource(new.run));calls={ast.unparse(n.func):n for n in ast.walk(tree) if isinstance(n,ast.Call)} + for name in ['random.seed','np.random.seed','torch.manual_seed','torch.cuda.manual_seed_all']:self.assertEqual(calls[name].args[0].value,1) + saves=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and ast.unparse(n.func)=='torch.save'];self.assertEqual(len(saves),1) + fields={k.arg:ast.literal_eval(k.value) for k in saves[0].args[0].keywords if k.arg in ['seed','role']};self.assertEqual(fields,dict(seed=1,role='last_epoch_only')) + def test_first_epoch_gate_not_all_folds_barrier(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'jobs/0').mkdir(parents=True);save(p/'contract.json',{});self.assertFalse(first_epoch_release(p,0));save(p/'jobs/0/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(p/'contract.json')));self.assertTrue(first_epoch_release(p,0));self.assertFalse(first_epoch_release(p,1));save(p/'global-stop.json',{}) + with self.assertRaises(RuntimeError):first_epoch_release(p,0) + def test_full_reserves_then_settled_cost_once(self): + self.assertEqual(project_total(self.c,{}, {})['projected_gpu_seconds'],118800) + d=project_total(self.c,{}, {0:dict(gpu_seconds_charged=20000)}) + self.assertEqual(d['projected_gpu_seconds'],116480);self.assertEqual(d['projected_phase3_gpu_seconds'],189480) + def test_validation_and_setup_in_projection(self): + o=dict(epoch=1,charged=100,max_epoch_seconds=80,max_validation_seconds=30) + d=project_cell(self.jobs[0],o);self.assertEqual(d['projected_gpu_seconds'],100+1.25*(199*80+20*30)+120) + def test_boundary_overrun_stops(self): + d=project_cell(self.jobs[0],dict(epoch=1,charged=100,max_epoch_seconds=100));self.assertFalse(d['within_cap']) + self.assertEqual(project_total(self.c,{0:d},{})['status'],'STOP_BUDGET_PROJECTION') + def test_parallel_dispatch_respects_held_gpus_and_identity(self): + self.assertEqual([j['id'] for j in dispatchable(self.jobs,[],set(GPUS[:2]))],[0,1]) + self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[dict(self.jobs[0],gpu_uuid=GPUS[0]),dict(self.jobs[1],gpu_uuid=GPUS[1])],set(GPUS))],[2,3]) + self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[],{GPUS[0],GPUS[3]})],[2,3]) + def test_gpu2_waits_for_prior_track_postfit(self): + with tempfile.TemporaryDirectory() as d: + r=Path(d);[(r/n).mkdir() for n in ['item','train','post']];c=dict(item1_recovery_root=str(r/'item'),track_a_training_root=str(r/'train'),track_a_postfit_root=str(r/'post')) + with patch('psutil.process_iter',return_value=[]): + self.assertEqual(available_gpus(c),{GPUS[0]});save(r/'item/complete.json',{});save(r/'train/complete.json',{}) + self.assertEqual(available_gpus(c),{GPUS[0],GPUS[1],GPUS[3]});save(r/'post/complete.json',{});self.assertEqual(available_gpus(c),set(GPUS)) + def test_busy_first_idle_second_does_not_consume_first_fold(self): + pending=copy.deepcopy(self.jobs);queries=[] + def query(u): + queries.append(u);return (2*2**30,50) if u==GPUS[0] else (5*2**20,0) + idle,observed=idle_eligible_gpus(set(GPUS[:2]),query) + assigned=dispatchable(pending,[],idle) + self.assertEqual(queries,GPUS[:2]);self.assertEqual([(j['id'],j['gpu_uuid']) for j in assigned],[(0,GPUS[1])]);self.assertEqual(pending,self.jobs);self.assertFalse(observed[0]['idle']) + def test_all_busy_leaves_pending_without_assignment(self): + pending=copy.deepcopy(self.jobs);idle,_=idle_eligible_gpus({GPUS[0]},lambda u:(200*2**20,0));self.assertEqual(dispatchable(pending,[],idle),[]);self.assertEqual(pending,self.jobs) + def test_ineligible_gpu_not_queried_and_bad_probe_rejected(self): + calls=[] + def query(u):calls.append(u);return 0,0 + idle,_=idle_eligible_gpus({GPUS[3]},query);self.assertEqual(calls,[GPUS[3]]);self.assertEqual(idle,{GPUS[3]}) + with self.assertRaises(ValueError):idle_eligible_gpus({GPUS[0]},lambda u:(float('nan'),0)) + def test_assignment_is_hash_bound_and_fold_specific(self): + with tempfile.TemporaryDirectory() as d: + r=Path(d);(r/'jobs/2').mkdir(parents=True);save(r/'contract.json',{});p=r/'jobs/2/assignment.json';save(p,dict(id=2,gpu_uuid=GPUS[3],contract_sha256=sha(r/'contract.json'))) + h=sha(p);self.assertEqual(assigned_job(r,self.c,2,h)['gpu_uuid'],GPUS[3]);save(p,dict(id=2,gpu_uuid=GPUS[0],contract_sha256=sha(r/'contract.json'))) + with self.assertRaises(ValueError):assigned_job(r,self.c,2,h) + def guard_exercise(self,peak=100,monitor=0): + with tempfile.TemporaryDirectory() as d: + q=Path(d);save(q/'contract.json',{});r=q/'jobs/0';r.mkdir(parents=True) + j=dict(id=0,gpu_uuid='synthetic-seed1',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,target_steps=400,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30) + def work(*args,**kwargs): + (r/'worker').mkdir();save(r/'worker/result.json',dict(status='COMPLETE',fold=0,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=monitor) + with patch.object(guard,'config',return_value=(dict(jobs=[j]),{})),patch.object(guard,'assigned_job',return_value=j),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',work),patch.object(guard.time,'sleep'): + rc=guard.run(q,0,'synthetic') + return rc,json.loads((r/'complete.json').read_text()) + def test_guard_clean_complete(self):self.assertEqual(self.guard_exercise()[0],0) + def test_guard_rejects_monitor_errors(self):self.assertIn('WORKER_MONITOR_ERRORS',self.guard_exercise(monitor=1)[1]['reasons']) + def test_guard_preserves_excess_recorded_peak(self):self.assertIn('RECORDED_VRAM_LIMIT',self.guard_exercise(peak=16*2**30)[1]['reasons']) + +if __name__=='__main__':unittest.main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py new file mode 100644 index 000000000..4f29e16d4 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py @@ -0,0 +1,62 @@ +"""Narrow CPU-only continuation delta checks; no datasets or CUDA execution.""" +import ast +import copy +import inspect +import unittest +from pathlib import Path +from phase3_seed1_common import project_cell, project_total + +TOOLS=Path(__file__).resolve().parent + +class ContinuationTests(unittest.TestCase): + def test_worker_guard_queue_only_entry_names_change(self): + for suffix in ['worker','guard','queue']: + original=(TOOLS/f'phase3_seed1_{suffix}.py').read_text() + new=(TOOLS/f'phase3_seed1_continuation_{suffix}.py').read_text() + restored=new.replace('phase3_seed1_continuation_common','phase3_seed1_common').replace('tools/phase3_seed1_continuation_guard.py','tools/phase3_seed1_guard.py').replace('tools/phase3_seed1_continuation_worker.py','tools/phase3_seed1_worker.py') + self.assertEqual(original,restored) + + def test_common_projection_idle_and_release_functions_unchanged(self): + import phase3_seed1_common as old + import phase3_seed1_continuation_common as new + for name in ['fold_view','project_cell','project_total','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']: + self.assertEqual(inspect.getsource(getattr(old,name)),inspect.getsource(getattr(new,name))) + + def test_duplicate_detection_includes_both_generations(self): + from phase3_seed1_continuation_common import execution_command + for prefix in ['phase3_seed1','phase3_seed1_continuation']: + for suffix in ['queue','guard','worker']: + self.assertTrue(execution_command(['python',f'tools/{prefix}_{suffix}.py'])) + self.assertFalse(execution_command(['python','pilot.py'])) + + def budget_fixture(self): + from prepare_phase3_seed1_continuation import remaining_contract + old=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200) + terminal=dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=269.53611274203286),dict(id=1,gpu_seconds_charged=269.5359442579793)],training_gpu_seconds=539.0720570000121) + original=copy.deepcopy(old);c=remaining_contract(old,terminal);self.assertEqual(old,original) + return c + + def test_full_pending_caps_include_old_cost_once(self): + c=self.budget_fixture();p=project_total(c,{}, {}) + self.assertEqual(p['status'],'PASS') + self.assertAlmostEqual(p['projected_gpu_seconds']+c['old_seed1_gpu_once'],167200,places=8) + self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200,places=8) + self.assertEqual(sum(j['gpu_seconds'] for j in c['jobs']),c['train_gpu_cap']) + self.assertEqual(c['postfit_gpu_reserve'],7200) + + def test_settled_new_cost_does_not_double_charge_old(self): + c=self.budget_fixture();terminal={i:dict(gpu_seconds_charged=10000+i) for i in range(5)} + p=project_total(c,{},terminal) + self.assertAlmostEqual(p['projected_gpu_seconds'],57210) + self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+539.0720570000121+57210) + self.assertEqual(p['status'],'PASS') + + def test_final_fold_or_total_overrun_stops(self): + c=self.budget_fixture();estimates={0:dict(projected_gpu_seconds=c['jobs'][0]['gpu_seconds']+.01)} + self.assertEqual(project_total(c,estimates,{})['status'],'STOP_BUDGET_PROJECTION') + observation=dict(epoch=2,charged=270,max_epoch_seconds=104.409218447,max_validation_seconds=0) + self.assertTrue(project_cell(c['jobs'][1],observation)['within_cap']) + observation['max_epoch_seconds']=140 + self.assertFalse(project_cell(c['jobs'][1],observation)['within_cap']) + +if __name__=='__main__':unittest.main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py new file mode 100644 index 000000000..35d1e2714 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py @@ -0,0 +1,91 @@ +"""CPU-only ledger/projection/observation tests for the new stopping rule.""" +import copy +import inspect +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch +import phase3_seed1_continuation_common as prior +import phase3_seed1_v2_common as new +import phase3_seed1_v2_observation as observation +from prepare_phase3_seed1_v2 import remaining_contract,OLD,PRIOR + +class V2Tests(unittest.TestCase): + def setUp(self): + original=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200) + def stopped(a,b):return dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=a),dict(id=1,gpu_seconds_charged=b)],training_gpu_seconds=a+b) + original_copy=copy.deepcopy(original) + self.c=remaining_contract(original,[(OLD,stopped(269.53611274203286,269.5359442579793)),(PRIOR,stopped(4317.711953648948,3625.5293972480576))]) + self.assertEqual(original,original_copy) + def est(self,v,charged=100):return dict(projected_gpu_seconds=v,actual_charged_gpu_seconds=charged) + def test_both_old_attempts_once_and_initial_budget(self): + p=new.project_total(self.c,{},{}) + self.assertEqual(p['status'],'PASS') + self.assertAlmostEqual(self.c['old_seed1_gpu_once'],8482.313407897018) + self.assertAlmostEqual(p['projected_gpu_seconds']+self.c['old_seed1_gpu_once'],167200) + self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200) + self.assertAlmostEqual(p['pending_reserve_gpu_seconds'],self.c['train_gpu_cap']/5) + def test_one_measured_fold_sets_pending_median(self): + p=new.project_total(self.c,{0:self.est(24000)},{},{0:100}) + self.assertEqual(p['projected_gpu_seconds'],5*24000+7200) + self.assertEqual(p['pending_reserve_gpu_seconds'],24000) + def test_even_median_and_two_started_count_once(self): + p=new.project_total(self.c,{0:self.est(20000),1:self.est(30000)},{},{0:1000,1:2000}) + self.assertEqual(p['pending_reserve_gpu_seconds'],25000) + self.assertEqual(p['projected_gpu_seconds'],20000+30000+3*25000+7200) + def test_total_exceed_stops_even_below_runaway(self): + p=new.project_total(self.c,{0:self.est(31000)},{},{0:100}) + self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION') + self.assertTrue(all(r['actual_within_runaway'] for r in p['rows'])) + def test_projected_above40000_does_not_alone_stop(self): + p=new.project_total(self.c,{0:self.est(45000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:100}) + self.assertEqual(p['status'],'PASS') + self.assertEqual(p['projected_gpu_seconds'],56200) + def test_actual_runaway_stops_within_total(self): + p=new.project_total(self.c,{0:self.est(10000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:40000.01}) + self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION') + self.assertTrue(p['total_projection_within_cap']);self.assertFalse(p['rows'][0]['actual_within_runaway']) + def test_terminal_actual_replaces_prior_estimate(self): + p=new.project_total(self.c,{0:self.est(45000)},{0:dict(gpu_seconds_charged=20000)},{}) + self.assertEqual(p['projected_gpu_seconds'],107200) + self.assertEqual(p['rows'][0]['basis'],'terminal_actual') + def test_active_unmeasured_not_free_and_actual_not_ignored(self): + p=new.project_total(self.c,{0:self.est(20000)},{},{0:100,1:100}) + initial=self.c['train_gpu_cap']/5 + self.assertEqual(p['rows'][1]['projected_gpu_seconds'],initial) + self.assertEqual(p['pending_reserve_gpu_seconds'],(20000+initial)/2) + q=new.project_total(self.c,{0:self.est(20000)},{},{0:35000}) + self.assertEqual(q['rows'][0]['projected_gpu_seconds'],35000) + with self.assertRaises(ValueError):new.project_total(self.c,{0:self.est(float('nan'))},{}) + with self.assertRaises(ValueError):new.project_total(self.c,{}, {},{5:100}) + def test_projection_math_unchanged(self): + j=self.c['jobs'][0];obs=dict(epoch=34,charged=3620,max_epoch_seconds=134.2113543249434,max_validation_seconds=15) + self.assertEqual(new.project_cell(j,obs)['projected_gpu_seconds'],prior.project_cell(j,obs)['projected_gpu_seconds']) + def test_scientific_entry_and_guard_only_names_changed(self): + b=Path(__file__).resolve().parent + for suffix in ['worker','guard']: + s=(b/f'phase3_seed1_v2_{suffix}.py').read_text().replace('phase3_seed1_v2_common','phase3_seed1_continuation_common').replace('phase3_seed1_v2_worker.py','phase3_seed1_continuation_worker.py') + self.assertEqual(s,(b/f'phase3_seed1_continuation_{suffix}.py').read_text()) + for name in ['fold_view','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']: + self.assertEqual(inspect.getsource(getattr(new,name)),inspect.getsource(getattr(prior,name))) + for prefix in ['phase3_seed1','phase3_seed1_continuation','phase3_seed1_v2']: + for suffix in ['worker','guard','queue']:self.assertTrue(new.execution_command(['python',f'tools/{prefix}_{suffix}.py'])) + def test_epoch_snapshot_marks_foreign_without_intervening(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d])) + response=SimpleNamespace(stdout='GPU-A, 10, python, 800\nGPU-A, 20, pilot.py, 900\nGPU-B, 30, python, 1000\n') + with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=response) as run: + r=observation.epoch_host_observation(root,0,1,'GPU-A') + self.assertEqual(r['status'],'OBSERVED');self.assertEqual([p['pid'] for p in r['gpu_processes'] if p['foreign_on_assigned_gpu']],[20]) + self.assertEqual(run.call_args.kwargs['timeout'],5) + self.assertTrue((root/'jobs/0/epoch-host-observations/001.json').exists()) + def test_snapshot_error_retained_observation_only(self): + with tempfile.TemporaryDirectory() as d: + with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',side_effect=subprocess.TimeoutExpired('nvidia-smi',5)): + r=observation.epoch_host_observation(Path(d),1,2,'GPU-A') + self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot') + +if __name__=='__main__':unittest.main() diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py new file mode 100644 index 000000000..c64df4841 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py @@ -0,0 +1,144 @@ +"""Track A random-split identity and cost gates; no array or target reads.""" +import json +import math +from pathlib import Path + +from phase2_h1_worker_v2 import checked, sha +from phase2_home_freeze_io import verify + +GPU_SECONDS = 162000 +PROTECTED_GPUS = { + 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', + 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36', +} + + +def read(path): + return json.loads(Path(path).read_text()) + + +def validate_roles(c, entry): + if len(c['parts']) != 615 or len(c['jobs']) != 5: + raise ValueError('Five original splits and 615 v4 parts required') + if c['original_failed'] != entry['original_failed']: + raise ValueError('Original failures changed') + for case, original in entry['H2_inputs'].items(): + f = original['features_and_targets'] + expected = dict(case=case, output=f['output'], nodes=f['nodes'], + t_set=f['t_scale_seconds'], output_sha256=f['output_sha256']) + if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']: + raise ValueError('Frozen native v4 input identity differs') + for j, original in zip(c['jobs'], entry['jobs']): + if j['id'] != original['split'] or j['split_seed'] != original['split_seed']: + raise ValueError('Original split identity changed') + for role in ('learning', 'test'): + if j[role] != original[role] or len(set(j[role])) != len(j[role]): + raise ValueError('Original membership/order changed: ' + role) + if j['validation'] or set(j['learning']) & set(j['test']): + raise ValueError('No new holdout or test leakage') + clock = read(checked(j['clock_path'], j['clock_sha256'])) + if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']): + raise ValueError('Clock learning membership differs') + if clock['groups'] != ['part:' + x for x in clock['parts']]: + raise ValueError('Original v4 equal-part clock groups differ') + if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0: + raise ValueError('Clock value differs') + if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']): + raise ValueError('Learning node count differs') + if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']): + raise ValueError('Protected GPU or epoch count') + + +def config(root): + root = Path(root) + c = read(root / 'contract.json') + verify(root, 'implementation') + if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS + or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200 + or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS): + raise PermissionError('Frozen Track A scope differs') + if c['protocol_ruling'] != '1794975093.432709': + raise PermissionError('Actual protocol ruling differs') + for p, h in {**c['source_sha256'], **c['code_sha256']}.items(): + checked(p, h) + if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']: + raise ValueError('Five job budgets must partition training cap') + entry = read(c['reuse_entry']) + validate_roles(c, entry) + import numpy as np + import torch + if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']: + raise ValueError('Original library versions differ') + return c, c + + +def view_for(c, job): + # The shared loader rejects every case outside this split's learning role. + return dict(c, folds=[job]) + + +def project_cell(job, observation): + epoch = observation['epoch'] + duration = observation['max_epoch_seconds'] + charged = observation['charged'] + if (not isinstance(epoch, int) or not 1 <= epoch <= 200 + or not math.isfinite(duration) or duration <= 0 + or not math.isfinite(charged) or charged < 0): + raise ValueError('Invalid full-epoch cost evidence') + projected = charged + 1.25 * (200 - epoch) * duration + 120 + return dict(id=job['id'], projected_gpu_seconds=projected, + within_cap=projected <= job['gpu_seconds']) + + +def project_total(c, estimates, terminal, active_charges=None): + """Pending jobs retain full reserved caps; terminal costs replace estimates.""" + active_charges = active_charges or {} + valid = {j['id'] for j in c['jobs']} + if (set(estimates) | set(terminal) | set(active_charges)) - valid: + raise ValueError('Unknown job in ledger') + total = c['postfit_gpu_reserve'] + rows = [] + for j in c['jobs']: + i = j['id'] + value = (terminal[i]['gpu_seconds_charged'] if i in terminal else + max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds']))) + if not math.isfinite(value) or value < 0: + raise ValueError('Invalid ledger value') + total += value + rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds'])) + return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows) + else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total, + postfit_reserve=c['postfit_gpu_reserve']) + + +def first_epoch_release(root, jobid): + if (root / 'global-stop.json').exists(): + raise RuntimeError('Track A global stop') + p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json' + if not p.exists(): + return False + r = read(p) + if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'): + raise RuntimeError('First-epoch cost gate rejected') + return True + + +def dispatchable(pending, active): + occupied = {j['gpu_uuid'] for j in active} + if len(occupied) != len(active) or len(active) > 2: + raise ValueError('Active GPU collision') + selected = [] + for j in pending: + if len(active) + len(selected) == 2: + break + if j['gpu_uuid'] in PROTECTED_GPUS: + raise ValueError('Item1 GPU reservation') + if j['gpu_uuid'] not in occupied: + selected.append(j) + occupied.add(j['gpu_uuid']) + return selected + + +def execution_command(args): + return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py', + 'track_a_h2_worker.py'} for a in args) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py new file mode 100644 index 000000000..d9d2c1d2b --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py @@ -0,0 +1,108 @@ +"""Build the independent Track A contract using JSON metadata only. Never launch.""" +import ast +import json +from pathlib import Path +import sys +import time + +from phase2_h1_worker_v2 import checked, save, sha +from track_a_h2_common import read, validate_roles + +BASE = Path(__file__).resolve().parents[1] +ENTRY = BASE / 'artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json' +ENTRY_SHA = '8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6' +RULING = '1794975093.432709' + + +def local_code_closure(paths): + found = set() + pending = list(paths) + while pending: + p = Path(pending.pop()).resolve() + if p in found: + continue + found.add(p) + for node in ast.walk(ast.parse(p.read_text())): + names = ([node.module] if isinstance(node, ast.ImportFrom) and node.module else + [a.name for a in node.names] if isinstance(node, ast.Import) else []) + for name in names: + q = BASE / 'tools' / (name.split('.')[0] + '.py') + if q.exists(): + pending.append(q) + return {str(p): sha(p) for p in sorted(found)} + + +def prepare(root): + cpu = time.process_time() + entry = read(checked(ENTRY, ENTRY_SHA)) + sources = {str(ENTRY): ENTRY_SHA} + for key in ('A_models_freeze', 'A_predictions_freeze', 'split_manifest', + 'preparation_manifest', 'geometry_manifest', 'source_h2_recipe'): + ref = entry[key] + checked(ref['path'], ref['sha256']) + sources[ref['path']] = ref['sha256'] + pilot = read(entry['source_h2_recipe']['path']) + h1root = BASE / 'artifacts/phase2-h1-labelled-recovery-2026-11-17' + h1models = read(h1root / 'models-terminal.json')['models'] + sources[str(h1root / 'models-terminal.json')] = sha(h1root / 'models-terminal.json') + parts = {} + geometry = {} + for case, row in entry['H2_inputs'].items(): + f = row['features_and_targets'] + parts[case] = dict(case=case, output=f['output'], nodes=f['nodes'], + t_set=f['t_scale_seconds'], output_sha256=f['output_sha256']) + geometry[case] = row['geometry'] + if geometry[case]['protocol'] != 'v4_random' or geometry[case]['nodes'] != f['nodes']: + raise ValueError('Native v4 geometry metadata mismatch') + jobs = [] + gpus = ['GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a'] + for original in entry['jobs']: + i = original['split'] + model = next(m for m in h1models if m['index'] == original['existing_H1_v4_job']) + if model['status'] != 'PASS': + raise ValueError('Original H1 clock unavailable') + clock = read(checked(model['clock_path'], model['clock_sha256'])) + sources[model['clock_path']] = model['clock_sha256'] + jobs.append(dict(id=i, split_seed=original['split_seed'], fold=i, + learning=original['learning'], test=original['test'], validation=[], + missing_train=original['missing_train'], missing_test=original['missing_test'], + clock_path=model['clock_path'], clock_sha256=model['clock_sha256'], c=clock['c'], + learning_nodes=sum(parts[x]['nodes'] for x in original['learning']), + learning_parts=len(original['learning']), validation_nodes=0, + target_steps=200 * len(original['learning']), gpu_uuid=gpus[i % 2], + worker_cpus=[8 + 2 * (i % 2)], guard_cpus=[28 + (i % 2)], + gpu_seconds=30960, setup_gpu_seconds=60, cpu_seconds=61920, + vram_bytes=15 * 2**30, rss_bytes=16 * 2**30, disk_bytes=8 * 2**30)) + code = local_code_closure([*[BASE / 'tools' / ('track_a_h2_' + name + '.py') + for name in ('common', 'worker', 'guard', 'queue', 'prepare')], + BASE / 'tools/test_track_a_h2_entry.py']) + # Pin original scientific functions against the already frozen H2 recipe. + for name in ('phase2_h2_data.py', 'phase2_h2_model.py', 'phase2_h2_pilot_worker.py', 'phase2_residual_math.py'): + p = BASE / 'tools' / name + pin = next(h for path, h in pilot['code_sha256'].items() if Path(path).name == name) + checked(p, pin) + c = dict(version='track-a-h2-v1', approval='1794973932.596749', protocol_ruling=RULING, + reuse_entry=str(ENTRY), comparison_contract=str(ENTRY), comparison_contract_sha256=ENTRY_SHA, + new_fits=5, A_refit=False, seed=0, epochs=200, no_retry=True, new_decodes=0, + model=pilot['model'], versions=pilot['versions'], parts=parts, geometry=geometry, jobs=jobs, + original_failed=entry['original_failed'], source_sha256=sources, code_sha256=code, + gpu_cap=162000, train_gpu_cap=154800, postfit_gpu_reserve=7200, + coordinator_cpus=[30, 31], aggregate_rss_bytes=40 * 2**30, aggregate_disk_bytes=48 * 2**30, + normalization='Per-split learning-only original 19 columns, FP64 population mean/std; separate FP64 Vf skip', + validation='None; no checkpoint selection; no test access in this training entry', + selection='Last epoch 200 only', clock_policy='Reuse frozen v4 H1 equal successful train-part clock', + postfit='Separate exact Assist review required before any test access; training never starts postfit', + metric=entry['metric'], expected_pairing_counts_provisional=[304, 311, 308, 309, 305]) + validate_roles(c, entry) + root.mkdir(parents=True, exist_ok=False) + save(root / 'contract.json', c) + save(root / 'preparation-receipt.json', dict(status='ENTRY_PREPARED_NOT_EXECUTED', + contract_sha256=sha(root / 'contract.json'), code_files=len(code), + clock_values_recomputed=False, scientific_arrays_read=False, new_fits_started=False, + prior_A_audit_rerun=False, learning_counts=[len(j['learning']) for j in jobs], + cpu_seconds=time.process_time() - cpu, protocol_ruling=RULING)) + print(json.dumps(read(root / 'preparation-receipt.json'))) + + +if __name__ == '__main__': + prepare(Path(sys.argv[1]).resolve()) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json new file mode 100644 index 000000000..9a8573c3a --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json @@ -0,0 +1,146 @@ +{ + "timestamp": "2026-11-18T08:43:50.686415+00:00", + "binding_sha256": "b5baee370b6cabf45ef495c6d4ca37949486b3ac346d6ae304d215ccd904fc9c", + "hash_audit": { + "code_files": 51, + "source_files": 407, + "local_implementation_files": 56, + "old_stop_files": [ + 108, + 192 + ], + "all_current_and_local_git_proofs": "PASS", + "old_guards_and_assignments_match": true + }, + "independent_tests": { + "count": 12, + "status": "PASS", + "failures": 0, + "errors": 0, + "wall_seconds": 0.022 + }, + "additional_projection_boundaries": [ + "measured median36000 dominates initial reserve for unmeasured active fold", + "unmeasured active actual35000 dominates initial reserve and measured20000; resulting started median27500" + ], + "config": "PASS; metadata/hash/version checks only", + "initial_projection": { + "status": "PASS", + "rows": [ + { + "id": 0, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 1, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 2, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 3, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + }, + { + "id": 4, + "projected_gpu_seconds": 30303.537318420596, + "actual_charged_gpu_seconds": 0, + "actual_within_runaway": true, + "basis": "pending_median_reserve" + } + ], + "projected_gpu_seconds": 158717.68659210298, + "phase3_prior_gpu_once": 80903.62940728385, + "projected_phase3_gpu_seconds": 239621.31599938683, + "postfit_reserve": 7200, + "pending_reserve_gpu_seconds": 30303.537318420596, + "actual_runaway_gpu_seconds": 40000, + "total_projection_within_cap": true + }, + "ledger": { + "old_attempts": [ + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18", + "gpu_seconds": 539.0720570000121 + }, + { + "root": "/home/frank14f/MeltMind-runs/phase3-seed1-continuation-2026-11-18", + "gpu_seconds": 7943.241350897006 + } + ], + "old_seed1_once": 8482.313407897018, + "new_training_cap": 151517.68659210298, + "new_total_cap": 158717.68659210298, + "postfit": 7200, + "phase3_prior": 80903.62940728385, + "cumulative_seed1_cap": 167200 + }, + "unchanged_science": "Worker and guard byte-identical after renaming imports/child entry; all job fields except CPU/GPU caps match original; six role/idle/release helpers unchanged", + "observation_gap": { + "reproducer": "/home/frank14f/optane/MeltMind/artifacts/meltassist-seed1-v2-review-2026-11-18/reproduce_observation_gap.py", + "sha256": "ace732bf9beaece291a56b342739a6c8e9e528b27c889132baacf0a8a38819cb", + "source_sha256": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "method": "Execute exact collect and active epoch-loop AST fragments against temporary synthetic histories, mocked process and observation callback; no GPU query or production files", + "cases": [ + { + "scenario": "exited_before_next_poll", + "last_observed": 199, + "completed": [ + 199, + 200 + ], + "expected": [ + 200 + ], + "actual": [], + "missing": [ + 200 + ] + }, + { + "scenario": "two_unseen_epoch_receipts", + "last_observed": 198, + "completed": [ + 199, + 200 + ], + "expected": [ + 199, + 200 + ], + "actual": [ + 200 + ], + "missing": [ + 199 + ] + } + ] + }, + "root_markers": { + "review.json": false, + "started.json": false, + "complete.json": false + }, + "publication": "Small package publication not independently verified in this review", + "science_executed": false, + "real_gpu_query": false, + "target_values_loaded": false, + "model_values_loaded": false, + "source_edits": false, + "git_writes": false +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py new file mode 100644 index 000000000..bfb9aa877 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py @@ -0,0 +1,60 @@ +"""Replay exact coordinator AST fragments using synthetic files/processes only.""" +import ast +import io +import json +from pathlib import Path +import tempfile +import time +from types import SimpleNamespace +from phase3_seed1_v2_common import project_cell, project_total +from phase2_h1_worker_v2 import sha, save + +source = Path('tools/phase3_seed1_v2_queue.py') +tree = ast.parse(source.read_text()) +run = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'run') +collect = next(n for n in run.body if isinstance(n, ast.FunctionDef) and n.name == 'collect') +epoch_loop = next(n for n in ast.walk(run) if isinstance(n, ast.For) + and isinstance(n.iter, ast.Name) and n.iter.id == 'active') + +def compiled(node): + return compile(ast.fix_missing_locations(ast.Module(body=[node], type_ignores=[])), str(source), 'exec') + +results = [] +for scenario in ['exited_before_next_poll', 'two_unseen_epoch_receipts']: + with tempfile.TemporaryDirectory() as td: + root = Path(td) + work = root / 'jobs/0' + history = work / 'worker/history' + history.mkdir(parents=True) + save(root / 'contract.json', {}) + save(work / 'assignment.json', {}) + save(work / 'complete.json', dict(status='PASS', monitor_errors=[], + worker=dict(monitor_errors=0), gpu_seconds_charged=1000)) + for epoch in [199, 200]: + save(history / f'{epoch:03d}.json', dict(epoch=epoch, train_epoch_seconds=1)) + v = dict(process=SimpleNamespace(poll=lambda: 0), start=time.monotonic(), + log=io.StringIO(), job=dict(gpu_uuid='SYNTHETIC-GPU')) + calls = [] + jobs = [dict(id=i, setup_gpu_seconds=60, gpu_seconds=40000, + validation_seconds_bound=1) for i in range(5)] + env = dict(root=root, active={0:v}, terminal={}, last_epoch={0:199 if scenario.startswith('exited') else 198}, + c=dict(jobs=jobs, train_gpu_cap=151517.68659210298, + postfit_gpu_reserve=7200, gpu_cap=158717.68659210298, + phase3_prior_gpu_once=80903.62940728385, phase3_gpu_cap=612000, + runaway_gpu_seconds=40000), estimates={}, + time=time, read=lambda p:json.loads(p.read_text()), save=save, sha=sha, + stop=lambda reason:None, active_charges=lambda:{0:1000}, + epoch_host_observation=lambda root,i,e,gpu:calls.append(e), + project_cell=project_cell, project_total=project_total) + exec(compiled(collect), env) + if scenario.startswith('exited'): + assert env['collect'](0,v) is True + exec(compiled(epoch_loop), env) + expected = [200] if scenario.startswith('exited') else [199,200] + assert calls != expected + results.append(dict(scenario=scenario,expected_epoch_observations=expected, + actual_epoch_observations=calls, + terminal_ids=list(env['terminal']), + missing=sorted(set(expected)-set(calls)))) +print(json.dumps(dict(source_sha256=sha(source),reproductions=results, + scientific_workload=False,real_gpu_query=False),indent=2)) diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json new file mode 100644 index 000000000..6073a7c58 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json @@ -0,0 +1,98 @@ +{ + "reviewed_commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "implementation_proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8", + "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + }, + "verdict": "BLOCKED", + "reviewer": "MeltAssist", + "timestamp": "2026-11-18T08:43:50.686415+00:00", + "review_scope": "Seed1 v2 stopping-rule, ledger and required epoch observation review; directly read Claude1794990750/0899/0914", + "evidence_path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-seed1-v2-review-2026-11-18/evidence.json", + "evidence_sha256": "219bc5bb74575ae880803a89bf58bb1ae65dd55dc618c3710680d9d8af99371c", + "findings": [ + { + "id": "V2-EPOCH-OBSERVATION-COVERAGE", + "priority": "P2", + "file": "tools/phase3_seed1_v2_queue.py", + "line": 115, + "related_lines": [ + 79, + 126, + 131, + 134 + ], + "title": "Drain completed-epoch observation receipts before removing a terminal fold; do not silently skip a backlog", + "requirement": "Approved v2 records GPU process/host observations for every completed epoch, retaining incomplete observation evidence without causal claims.", + "observed": "The loop collects exited guards first and deletes their active entry. The observation loop only visits remaining active entries and only rows[-1]. Thus epoch200 can disappear when the guard exits before the next poll; multiple unseen completed epochs collapse to the latest. Exact coordinator-fragment CPU replay reproduced both omissions. No actual scientific run was used.", + "requested_fix": "Track observation coverage separately from cost-projection progress. Before terminal removal/global-stop cleanup, account for every completed-epoch receipt; handle all previously unseen epochs, not just latest. Record true snapshot time and explicitly mark delayed/unobserved history OBSERVATION_INCOMPLETE or equivalent rather than invent past GPU occupancy. Preserve projection formula, actual charging, idle dispatch and foreign-process nonintervention. Add terminal-before-poll and multi-receipt-backlog synthetic coverage." + } + ], + "passed_checks": [ + "Old539.0720570000121+7943.241350897006=8482.313407897018 counted once inside167200; new total158717.68659210298/train151517.68659210298+7200; phase3prior80903.62940728385 already includes both old attempts", + "Total projection uses started projections/terminal actual plus pending started-median reservations; initial train/5; active unmeasured max(actual,initial,measured median), then contributes to started median; no double addition of terminal actual", + "Measured max-epoch1.25/validation/setup formula unchanged; projection>40000 does not independently STOP; actual40000 guard remains with conservative setup/termination margins", + "Observation subprocess uses5s timeout, preserves query/identity errors as OBSERVATION_INCOMPLETE; return value not used by dispatch or projection and elapsed collection time is charged", + "Three-generation duplicate detection, shared GPU locks, actual-idle dispatch/GPU2 priority, scientific recipe and CPU/RSS/VRAM/disk protections remain", + "Independent12tests and two additional unmeasured-reserve boundary checks PASS; config/initial projection PASS;51code/407source/56implementation and both old108/192freeze-file proofs verified" + ], + "limitations": [ + "This is an observation coverage blocker, not a ledger failure, scientific failure or evidence of host contention", + "Prior epoch34 slowdown cause remains unproven; current or delayed snapshots cannot establish historical causality", + "Original stopped coordinator CPU accounting gaps preserved, not estimated", + "No launch or result claimed; old PASS cannot substitute for a new exact v2 verdict; postfit remains separately reviewed", + "No remote publication raw-byte verification claimed; local immutable identity verified" + ] +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json new file mode 100644 index 000000000..910e61c75 --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json @@ -0,0 +1,291 @@ +{ + "original_commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "preserved_files": [ + { + "original": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json", + "sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c" + }, + { + "original": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", + "sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493" + }, + { + "original": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json", + "sha256": "af9531defc4289d021eec3809d87ba4f7497927f94be78795027ca9248f21f8f" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py", + "sha256": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/gre24_models.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py", + "sha256": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py", + "sha256": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py", + "sha256": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py", + "sha256": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py", + "sha256": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py", + "sha256": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py", + "sha256": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py", + "sha256": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py", + "sha256": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py", + "sha256": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py", + "sha256": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py", + "sha256": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py", + "sha256": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py", + "sha256": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py", + "sha256": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py", + "sha256": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py", + "sha256": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py", + "sha256": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py", + "sha256": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py", + "sha256": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py", + "sha256": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py", + "sha256": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py", + "sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py", + "sha256": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py", + "sha256": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py", + "sha256": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py", + "sha256": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py", + "sha256": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_queue.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py", + "sha256": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py", + "sha256": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_continuation.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py", + "sha256": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_continuation.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py", + "sha256": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py", + "sha256": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py", + "sha256": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py", + "sha256": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py", + "sha256": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_queue.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py", + "sha256": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase3_seed1_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py", + "sha256": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py", + "sha256": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py", + "sha256": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py", + "sha256": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/s2_models/network.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py", + "sha256": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py", + "sha256": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py", + "sha256": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py", + "sha256": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py", + "sha256": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py", + "sha256": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py", + "sha256": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6" + }, + { + "original": "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py", + "sha256": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5" + }, + { + "original": "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md", + "sha256": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + }, + { + "original": "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-entry-2026-11-18.md", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md", + "sha256": "2ed35f7b8cb34fab947db736c6f13c4aaecf48513fbeb65d7b97aa9a31e93158" + }, + { + "original": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log", + "sha256": "65083e1e5ffb5b2514a2eabc0b83fd3ff4430e01530ff0b15c0478a26bf826bb" + } + ], + "review_hashes": { + "review.json": "fe873785b734467decd69ac1947c05b64a610361dd6f7f2226ff14964eccfa45", + "evidence.json": "219bc5bb74575ae880803a89bf58bb1ae65dd55dc618c3710680d9d8af99371c", + "reproduce_observation_gap.py": "ace732bf9beaece291a56b342739a6c8e9e528b27c889132baacf0a8a38819cb" + }, + "no_original_v2_launch": true +} diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log new file mode 100644 index 000000000..77ea2bcdb --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log @@ -0,0 +1,22 @@ +test_delayed_single_receipt_is_incomplete_without_query (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_fresh_single_receipt_has_real_snapshot_times_and_hash (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_global_stop_poll_covers_completed_receipt (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_terminal_before_next_poll_keeps_epoch200 (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_two_unseen_receipts_each_covered_no_historical_snapshot (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_active_unmeasured_not_free_and_actual_not_ignored (test_phase3_seed1_v2.V2Tests) ... ok +test_actual_runaway_stops_within_total (test_phase3_seed1_v2.V2Tests) ... ok +test_both_old_attempts_once_and_initial_budget (test_phase3_seed1_v2.V2Tests) ... ok +test_epoch_snapshot_marks_foreign_without_intervening (test_phase3_seed1_v2.V2Tests) ... ok +test_even_median_and_two_started_count_once (test_phase3_seed1_v2.V2Tests) ... ok +test_one_measured_fold_sets_pending_median (test_phase3_seed1_v2.V2Tests) ... ok +test_projected_above40000_does_not_alone_stop (test_phase3_seed1_v2.V2Tests) ... ok +test_projection_math_unchanged (test_phase3_seed1_v2.V2Tests) ... ok +test_scientific_entry_and_guard_only_names_changed (test_phase3_seed1_v2.V2Tests) ... ok +test_snapshot_error_retained_observation_only (test_phase3_seed1_v2.V2Tests) ... ok +test_terminal_actual_replaces_prior_estimate (test_phase3_seed1_v2.V2Tests) ... ok +test_total_exceed_stops_even_below_runaway (test_phase3_seed1_v2.V2Tests) ... ok + +---------------------------------------------------------------------- +Ran 17 tests in 0.052s + +OK diff --git a/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log new file mode 100644 index 000000000..15f410b7b --- /dev/null +++ b/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log @@ -0,0 +1,23 @@ +test_delayed_single_receipt_is_incomplete_without_query (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_fresh_single_receipt_has_real_snapshot_times_and_hash (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_global_stop_poll_covers_completed_receipt (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_terminal_before_next_poll_keeps_epoch200 (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_two_unseen_receipts_each_covered_no_historical_snapshot (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_worker_disappearing_between_poll_and_snapshot_is_incomplete (test_phase3_seed1_v2.ObservationCoverageTests) ... ok +test_active_unmeasured_not_free_and_actual_not_ignored (test_phase3_seed1_v2.V2Tests) ... ok +test_actual_runaway_stops_within_total (test_phase3_seed1_v2.V2Tests) ... ok +test_both_old_attempts_once_and_initial_budget (test_phase3_seed1_v2.V2Tests) ... ok +test_epoch_snapshot_marks_foreign_without_intervening (test_phase3_seed1_v2.V2Tests) ... ok +test_even_median_and_two_started_count_once (test_phase3_seed1_v2.V2Tests) ... ok +test_one_measured_fold_sets_pending_median (test_phase3_seed1_v2.V2Tests) ... ok +test_projected_above40000_does_not_alone_stop (test_phase3_seed1_v2.V2Tests) ... ok +test_projection_math_unchanged (test_phase3_seed1_v2.V2Tests) ... ok +test_scientific_entry_and_guard_only_names_changed (test_phase3_seed1_v2.V2Tests) ... ok +test_snapshot_error_retained_observation_only (test_phase3_seed1_v2.V2Tests) ... ok +test_terminal_actual_replaces_prior_estimate (test_phase3_seed1_v2.V2Tests) ... ok +test_total_exceed_stops_even_below_runaway (test_phase3_seed1_v2.V2Tests) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 0.055s + +OK diff --git a/docs/collaboration-stage-review-policy.md b/docs/collaboration-stage-review-policy.md index 9bc02e7a4..1f41812c6 100644 --- a/docs/collaboration-stage-review-policy.md +++ b/docs/collaboration-stage-review-policy.md @@ -665,3 +665,6 @@ Seed1单次接续实际启动:Assist新PASS5c8359c9按75eff37b/e90bc1b5完整 ## Seed1 v2明确规则和绑定送审(2026-11-18) Claude1794990899/0914明确:pending预留started当前投影median,无实测时剩余train/5;40000按实际charged,不按单折投影,合计与postfit对158717.686592核。旧8482.313408在累计167200内一次;phase3prior80903.629407已含两次旧账。慢轮原因未证已被Claude接受。新独立c76719ff/95ff82b9绑定含12CPU合成PASS、epoch观察仅记录,送Assist停止逻辑/账本窄审;当前交付是绑定posted,不是已审或已启动。两旧STOP保持,若v2最终超总帽则停报、无进一步seed1接续。 + + +Seed1 v2同次独立审阅发现观测覆盖P2(Assist1794991467,BLOCKEDfe873785),非预算或科学失败。原绑定/56文件/回执不改而保存快照。最小三文件修复已固定8603801c/c8077c38,18CPU检查含终态早于poll、双receipt积压、global-stop与worker消失;仅修观察coverage,已通过的账本/停止公式保持。新实际review仍须精确五字段绑定;当前待同次复核,不重复主审、不启动训练或干扰Track A。 diff --git a/docs/phase2-live-handoff-2026-11-17.md b/docs/phase2-live-handoff-2026-11-17.md index fdd15c6e0..21912a496 100644 --- a/docs/phase2-live-handoff-2026-11-17.md +++ b/docs/phase2-live-handoff-2026-11-17.md @@ -1,3 +1,7 @@ +## 最新 v2观测覆盖阻断已修复固定,待同次复核(2026-11-18) + +Assist原BLOCKEDfe873785仅V2-EPOCH-OBSERVATION-COVERAGE:terminal先collect移除及rows[-1]导致漏最后/积压epoch;原预算/ledger/科学通过。原56文件/绑定/复现及BLOCKED完整保存在artifacts/phase3-seed1-v2-observation-fix-2026-11-18。只改observation/queue/test三文件,新implementation8603801c/contractc8077c38/proofbaa0f87e/comparisoncba51291/51codes、407sources PASS,18CPU检查PASS。独立coverage游标、终态/停止清理前逐条排空;迟到/积压/terminal标INCOMPLETE不伪造历史,当前snapshot真实起止,worker消失显式INCOMPLETE。common预算/投影/idle及worker/guard不改。新binding已请求TRON转Assist同次复核,原BLOCKED不得启动;无review采用/started/launch。报告docs/phase3-seed1-v2-observation-fix-2026-11-18.md。两旧STOP与Track A不变,下方原送审c76719ff为历史。 + ## 最新 seed1 v2独立绑定已送逻辑窄审,未启动(2026-11-18) Claude1794990750/0899/0914已实读:同累计167200,不增加预算,两个旧STOP保持;新root `/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18`,implementationc76719ff/contract95ff82b9/proof9991e81e/comparisoncba51291/51codes/407sources,12CPU检查和config/初始投影PASS。已请求TRON转Assist实际新五字段review/evidence,当前无review/started/训练。旧8482.313408只扣一次,新total158717.686592/train151517.686592+post7200,phase3prior80903.629407。pending用started当前投影median,未测前train/5;active未测保守max(actual,initial,measuredmedian)并纳入median;40000只actual runaway不按单折projection停,原1.25公式保持。每epoch5s有界GPU进程快照仅观察,错误保留;科学worker/guard只改名字。报告docs/phase3-seed1-v2-entry-2026-11-18.md。此前continuation仍终态,完整raw d3c94a00已远端ref核验;不重启旧队列或以旧PASS代新审,不自动后处理。Track A独立原队列继续。 diff --git a/docs/phase3-seed1-v2-observation-fix-2026-11-18.md b/docs/phase3-seed1-v2-observation-fix-2026-11-18.md new file mode 100644 index 000000000..990daf2bc --- /dev/null +++ b/docs/phase3-seed1-v2-observation-fix-2026-11-18.md @@ -0,0 +1,13 @@ +# Same-review repair: completed-epoch observation coverage + +MeltAssist's actual review SHAfe873785b734467decd69ac1947c05b64a610361dd6f7f2226ff14964eccfa45 blocked original v2c76719ff/95ff82b9 solely on V2-EPOCH-OBSERVATION-COVERAGE. Budget, total projection, actual40000 safeguard, old-cost accounting, active-unmeasured reserve and scientific recipe passed. Review evidence SHA219bc5bb74575ae880803a89bf58bb1ae65dd55dc618c3710680d9d8af99371c and reproduction SHAace732bf9beaece291a56b342739a6c8e9e528b27c889132baacf0a8a38819cb were independently checked. This is a code-review finding, not automatic-approval rejection or scientific failure. + +The original BLOCKED, exact56-file implementation, contract/proof/binding and reproduction are preserved byte-for-byte under `artifacts/phase3-seed1-v2-observation-fix-2026-11-18`. Original local Git commit and remote publicationbf7363bf remain. No v2 training/started/launch existed before repair. + +Only three code files change: v2 observation helper, queue integration and tests. Common budget/project/dispatch helpers, worker, guard, source dataset pins, scientific comparison and costs are unchanged. Observation coverage uses a separate per-fold set, not the last projected epoch. Every poll accounts for all unseen completed-epoch receipts; collect drains before deleting a terminal fold. Global-stop and exception cleanup drain before termination/removal, with explicit coverage-error evidence if malformed input prevents coverage. The latest-epoch cost projection cursor and formula stay unchanged. + +Records are keyed by completed epoch and hash-bind its receipt. Terminal, stopping, multi-receipt backlog and receipts older than5s produce OBSERVATION_INCOMPLETE with a specific cause, actual record timestamp, receipt mtime/hash, no GPU query and null snapshot times. This5s distinction is observation classification only, not a compute budget or stopping threshold. Fresh live singleton receipts get a current bounded GPU snapshot with actual start/end timestamps. If the assigned worker disappears between poll and snapshot, the snapshot is explicitly incomplete; current observations never claim historical occupancy. Existing observation records are checked and retained, not overwritten; repeated scans do not duplicate them. Foreign processes are only observed and never modified. + +The two exact reported failure modes are exercised using the coordinator's actual observe/collect/poll AST fragments: terminal-before-poll preserves epoch200 before active deletion; a backlog199/200 produces both records. Added cases cover global-stop, delayed single receipt, fresh snapshot timing/hash and worker disappearance during observation. Original12 budget/science checks plus6 coverage cases =18CPU checks PASS. Initial17-check pass is also retained; the final extra race case and classification correction motivated the18-check rerun. No actual GPU query, labels, model arrays or scientific workload was used. + +The new binding is submitted for the same narrow review. The old BLOCKED is not a usable launch authorization. No new main review, scientific computation, Track A changes or modification of either old seed1 STOP is implied. Actual new exact PASS is required before any v2 launch. Slow epoch34's cause remains unproven; previous stopped-coordinator CPU gaps remain unfilled. diff --git a/home-runs/phase3-seed1-v2-2026-11-18/contract.json b/home-runs/phase3-seed1-v2-2026-11-18/contract.json index 176da7e52..bfd988e58 100644 --- a/home-runs/phase3-seed1-v2-2026-11-18/contract.json +++ b/home-runs/phase3-seed1-v2-2026-11-18/contract.json @@ -3363,8 +3363,8 @@ "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "8d6ef5031f253f2534e18297c9c4203a4ea76eb57abf83d5c3edd07240110707", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "4d3bb0c4f3110effaa44cb34d3ced186582d5978607c4fea158ebf505bb157f2", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", @@ -3373,7 +3373,7 @@ "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", - "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "4dff4943d53ff121dfca829474b31f1abe89441e3e5b3bcba53c3e5465b4b500", "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", @@ -3445,5 +3445,6 @@ "runaway_gpu_seconds": 40000, "epoch_foreign_gpu_observation": "bounded observation-only snapshot after each completed epoch; errors retained, no causal or scheduling conclusion", "accounting": "Runtime counters new-v2 only; cumulative seed1 adds old_attempts sum once; phase3 prior already contains both old attempts. Per-fold40000 is actual runaway, not partitioned reserve.", - "remaining_caps_final": true + "remaining_caps_final": true, + "observation_coverage_fix": "Separate covered-epoch set; drain all completed receipts before terminal/global-stop removal; delayed/backlog/terminal snapshots explicitly incomplete; do not fabricate historical occupancy" } diff --git a/home-runs/phase3-seed1-v2-2026-11-18/expected-review-binding.json b/home-runs/phase3-seed1-v2-2026-11-18/expected-review-binding.json index 86b1edfdd..fef03ee52 100644 --- a/home-runs/phase3-seed1-v2-2026-11-18/expected-review-binding.json +++ b/home-runs/phase3-seed1-v2-2026-11-18/expected-review-binding.json @@ -1,7 +1,7 @@ { - "reviewed_commit": "c76719ffe61934a74edffd201837ade0b24264e0", - "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", - "implementation_proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8", + "reviewed_commit": "8603801c4644068db5102227878e1af28c3b5f5e", + "contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981", + "implementation_proof_sha256": "baa0f87e82ff1cb36e6c09ee7e4d0fabe96808a3833776196bd21e53b7571d17", "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", "code_sha256": { "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", @@ -19,8 +19,8 @@ "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "8d6ef5031f253f2534e18297c9c4203a4ea76eb57abf83d5c3edd07240110707", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "4d3bb0c4f3110effaa44cb34d3ced186582d5978607c4fea158ebf505bb157f2", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", @@ -29,7 +29,7 @@ "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", - "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "4dff4943d53ff121dfca829474b31f1abe89441e3e5b3bcba53c3e5465b4b500", "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", diff --git a/home-runs/phase3-seed1-v2-2026-11-18/implementation-authorization.json b/home-runs/phase3-seed1-v2-2026-11-18/implementation-authorization.json index 42e220e40..6b7045d82 100644 --- a/home-runs/phase3-seed1-v2-2026-11-18/implementation-authorization.json +++ b/home-runs/phase3-seed1-v2-2026-11-18/implementation-authorization.json @@ -1,4 +1,4 @@ { - "contract_sha256": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", - "proof_sha256": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8" + "contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981", + "proof_sha256": "baa0f87e82ff1cb36e6c09ee7e4d0fabe96808a3833776196bd21e53b7571d17" } diff --git a/home-runs/phase3-seed1-v2-2026-11-18/implementation-git-freeze.json b/home-runs/phase3-seed1-v2-2026-11-18/implementation-git-freeze.json index 7cb699cde..aea6b48d8 100644 --- a/home-runs/phase3-seed1-v2-2026-11-18/implementation-git-freeze.json +++ b/home-runs/phase3-seed1-v2-2026-11-18/implementation-git-freeze.json @@ -1,9 +1,10 @@ { - "commit": "c76719ffe61934a74edffd201837ade0b24264e0", + "commit": "8603801c4644068db5102227878e1af28c3b5f5e", "files": { - "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981", "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "af9531defc4289d021eec3809d87ba4f7497927f94be78795027ca9248f21f8f", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json": "7d0c5a4b18713e1593477fa9990bbbe8b0b68521b1aa41ffe212bdc880e14272", "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", @@ -19,8 +20,8 @@ "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", - "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_observation.py": "8d6ef5031f253f2534e18297c9c4203a4ea76eb57abf83d5c3edd07240110707", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py": "4d3bb0c4f3110effaa44cb34d3ced186582d5978607c4fea158ebf505bb157f2", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", "/home/frank14f/optane/MeltMind/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", @@ -29,7 +30,7 @@ "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", - "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/tools/test_phase3_seed1_v2.py": "4dff4943d53ff121dfca829474b31f1abe89441e3e5b3bcba53c3e5465b4b500", "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", "/home/frank14f/optane/MeltMind/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", @@ -56,12 +57,78 @@ "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073", "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-entry-2026-11-18.md": "2ed35f7b8cb34fab947db736c6f13c4aaecf48513fbeb65d7b97aa9a31e93158", - "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "65083e1e5ffb5b2514a2eabc0b83fd3ff4430e01530ff0b15c0478a26bf826bb" + "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-observation-fix-2026-11-18.md": "912530090480c24ff656125efa8b22b4dca6acbf4a2df54465cb415eb1685b0a", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log": "154ab4e0aea80e07d1b4189529e5d471aa826db642897ffc8d0ce2a18398d678", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json": "966d6e0ba2423509673c6aebe952f4ab1f58f9863b81c9e07aedea252f16eee7", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log": "323c486b38959b0f3be73ca49da75baaaf9bb91f57f77b9f0c1f20c2fb01a65b", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json": "9991e81ed2d8236d54dbd37e3e2239de930f26a3ddeca80a5ddbd15176f9c6b8", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json": "b5baee370b6cabf45ef495c6d4ca37949486b3ac346d6ae304d215ccd904fc9c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json": "c9d6cb23d4f6c7ee98a102431f291981638b97a4a51eb348a2d1c079b3e2fc24", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "af9531defc4289d021eec3809d87ba4f7497927f94be78795027ca9248f21f8f", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json": "95ff82b95a8a5e6b46f798ee4f0bc9e503201092437b80db5281b0e0001fd08c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py": "f6415a4f7d4778ae79c5886f0ff1d6080b4ef32f493080e4bcb626ed7c7fd3e5", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py": "5c9a27e6abf3d16c1f2094c5f5e0ac77c906595a34ab32e419b0258f76c6256b", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py": "2eec8859f97c47e3b5e6424f042384079f45303fba4411064b442c67e0c2c253", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py": "a7b2b4c77b8646ab0b6616f1065390f91d2cc7ec386d52dd119cda2822d57c95", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py": "d365f3ee0f8a85382f8e51a2769a04a738b14959d97aa081a95e4f0aac2a5d53", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py": "eed47b0cfba0861700f5e1731051bae92065e9bedec727a4f15e6f5c77ce367c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py": "c394b7b8b12ba316910ac00abeba7ba0632a1d240d8e3fd03ddbf0cbc3b66ba1", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py": "dc57d2912a26977eb2a7f9a3ca81f59ac7ad28a3cdc69d3f8a00ed08399fd2a0", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py": "c8a0e9d7e9487d582e977b24084ebae4b6864d71dd08df0717304187ff03b0da", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py": "740da12c8bd36d812b281735cd2bde752d2c53dd2ce41f33cef7a7ae8f8df861", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py": "3493cec3eec50c5dcfd59a7236d6502b3ce4a364ee50998dbdf3c2c698f8198c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py": "0981d4ebd9444f9ef7bbc1b68cfb1aebfd39b2d799df73f81bc7c5f6ca1cf27b", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py": "5912e88b38c18cc9c895be51e6dfc932c7598f97ff58563b61bb482edb3893d3", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py": "ee858a52878032b49601e56e98e224640c62f86523495d85ceba644ce0c86172", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py": "03f33f3c0d30a3b51602a46422ecc8a0392bdc9435f335a90bcc1c1084a3f381", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py": "5a07472a2d99c03daa6915fbc233f35f0251caac6667adc248d177bb52bf722e", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py": "3c0289a06567a90077f8bb715ecf5e1e9f325338cc8b555c0a087488ff87e758", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py": "783ad2ce470a5453a238ca9c6699b30eec51cc92ef8c5ae6829304d1ff211d9c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md": "2ed35f7b8cb34fab947db736c6f13c4aaecf48513fbeb65d7b97aa9a31e93158", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "65083e1e5ffb5b2514a2eabc0b83fd3ff4430e01530ff0b15c0478a26bf826bb", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json": "219bc5bb74575ae880803a89bf58bb1ae65dd55dc618c3710680d9d8af99371c", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py": "ace732bf9beaece291a56b342739a6c8e9e528b27c889132baacf0a8a38819cb", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json": "fe873785b734467decd69ac1947c05b64a610361dd6f7f2226ff14964eccfa45" }, "git_paths": { "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "home-runs/phase3-seed1-v2-2026-11-18/contract.json", "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json": "home-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json", "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py", "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py", "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py", @@ -114,7 +181,72 @@ "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py", "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md", "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-entry-2026-11-18.md": "docs/phase3-seed1-v2-entry-2026-11-18.md", - "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log" + "/home/frank14f/optane/MeltMind/docs/phase3-seed1-v2-observation-fix-2026-11-18.md": "docs/phase3-seed1-v2-observation-fix-2026-11-18.md", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-02.log", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/preservation.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/tests-attempt-01.log", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-git-freeze.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/expected-review-binding.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-binding/implementation-authorization.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/preparation-audit.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/contract.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/home-runs/phase3-seed1-v2-2026-11-18/comparison-contract.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_prepare.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_local_freeze_io.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_prepare.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_prepare.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_residual_math.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_queue.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase2_h2_s2.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_gpu_idle_gate.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_launch.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_monitoring.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_main_memory_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_gate_outcome_review.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_fixed_io.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s2_queue.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_pilot_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/test_phase3_seed1_continuation.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_model.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/track_a_h2_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/gre24_models.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_v2_observation.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_queue.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_queue.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_common.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/freeze_g1_components.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_data.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_home_freeze_io.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/prepare_phase3_seed1_continuation.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_precision_continuation_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h1_worker_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_pilot_train.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_recovery_gpu_supervisor.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase2_h2_s1_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/phase3_seed1_continuation_guard.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/tools/s2_models/network.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase2-h2-s2-execution-contract-2026-11-17.md", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/docs/phase3-seed1-v2-entry-2026-11-18.md", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-implementation/artifacts/phase3-seed1-v2-acceptance-2026-11-18/attempt-01.log", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/evidence.json", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/reproduce_observation_gap.py", + "/home/frank14f/optane/MeltMind/artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json": "artifacts/phase3-seed1-v2-observation-fix-2026-11-18/original-review/review.json" }, "remote_published": false, "local_repository": "/tmp/meltmind-phase2-local-fixed-20261117" diff --git a/home-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json b/home-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json new file mode 100644 index 000000000..062909729 --- /dev/null +++ b/home-runs/phase3-seed1-v2-2026-11-18/observation-fix-audit.json @@ -0,0 +1,21 @@ +{ + "status": "FIXED_PENDING_SAME_REVIEW", + "unix": 1794991781.9360166, + "old_implementation": "c76719ffe61934a74edffd201837ade0b24264e0", + "old_blocked_sha256": "fe873785b734467decd69ac1947c05b64a610361dd6f7f2226ff14964eccfa45", + "changed_code_files": [ + "tools/phase3_seed1_v2_observation.py", + "tools/phase3_seed1_v2_queue.py", + "tools/test_phase3_seed1_v2.py" + ], + "unchanged_code_files": 48, + "source_pins": "PASS", + "source_files": 407, + "comparison_contract_sha256": "cba51291811c224966ccab82ff64a9b0690c83eaf2482316853ec9c656322493", + "cpu_tests": 18, + "tests_pass": true, + "tests_log_sha256": "154ab4e0aea80e07d1b4189529e5d471aa826db642897ffc8d0ce2a18398d678", + "budget_projection_science_unchanged": true, + "old_snapshot_preservation_sha256": "966d6e0ba2423509673c6aebe952f4ab1f58f9863b81c9e07aedea252f16eee7", + "started": false +} diff --git a/tools/phase3_seed1_v2_observation.py b/tools/phase3_seed1_v2_observation.py index dd41ca7c8..f3d142ea4 100644 --- a/tools/phase3_seed1_v2_observation.py +++ b/tools/phase3_seed1_v2_observation.py @@ -1,15 +1,23 @@ """Epoch-boundary host observations only; no scheduling or causal decisions.""" import csv import io +import json import os from pathlib import Path import subprocess import time import psutil -from phase2_h1_worker_v2 import save +from phase2_h1_worker_v2 import save,sha -def epoch_host_observation(root,fold,epoch,gpu_uuid): +def epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=None,incomplete_reason=None): record=dict(unix=time.time(),fold=fold,completed_epoch=epoch,assigned_gpu_uuid=gpu_uuid,observation_only=True,sampling='After the coordinator reads the completed-epoch receipt; not continuous monitoring and not causal proof',gpu_processes=[],errors=[]) + if receipt is not None: + receipt=Path(receipt) + record.update(receipt_path=str(receipt),receipt_sha256=sha(receipt),receipt_mtime_unix=receipt.stat().st_mtime,receipt_age_seconds=max(0,record['unix']-receipt.stat().st_mtime)) + if incomplete_reason is not None: + record.update(status='OBSERVATION_INCOMPLETE',incomplete_reason=incomplete_reason,snapshot_started_unix=None,snapshot_finished_unix=None,own_worker_pids=[],historical_gpu_occupancy_observed=False) + return write_observation(root,fold,epoch,record) + record['snapshot_started_unix']=time.time() try:record['load_average']=list(os.getloadavg()) except OSError as exc:record['errors'].append(dict(stage='load_average',type=type(exc).__name__,reason=str(exc))) own=set() @@ -26,9 +34,43 @@ def epoch_host_observation(root,fold,epoch,gpu_uuid): uuid,pid,name,memory=[s.strip() for s in row];pid=int(pid) record['gpu_processes'].append(dict(gpu_uuid=uuid,pid=pid,process_name=name,used_memory_mib=memory,own_v2_worker=pid in own,on_assigned_gpu=uuid==gpu_uuid,foreign_on_assigned_gpu=uuid==gpu_uuid and pid not in own)) except Exception as exc:record['errors'].append(dict(stage='gpu_process_snapshot',type=type(exc).__name__,reason=str(exc))) + if not any(p['own_v2_worker'] and p['on_assigned_gpu'] for p in record['gpu_processes']): + record['errors'].append(dict(stage='epoch_worker_visibility',reason='Assigned v2 worker not visible at snapshot; historical occupancy not observed')) record['own_worker_pids']=sorted(own);record['status']='OBSERVED' if not record['errors'] else 'OBSERVATION_INCOMPLETE' + record['snapshot_finished_unix']=time.time() + record['historical_gpu_occupancy_observed']=False + return write_observation(root,fold,epoch,record) + + +def write_observation(root,fold,epoch,record): p=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json' p.parent.mkdir(parents=True,exist_ok=True) if p.exists():raise FileExistsError('Preserve original epoch observation') save(p,record) return record + + +def drain_epoch_observations(root,fold,gpu_uuid,covered,terminal=False,stopping=False): + """Account for each completed receipt independently of the cost cursor. + + Terminal, stopping, batched or >5s-old receipts get explicit missing-history + records without GPU queries. A fresh live singleton gets a current snapshot; + its true times never claim coverage of earlier epoch occupancy. + """ + history=root/'jobs'/str(fold)/'worker/history' + receipts=[] + for p in sorted(history.glob('*.json')): + row=json.loads(p.read_text());epoch=row['epoch'] + if type(epoch) is not int or not 1<=epoch<=200 or p.stem!=f'{epoch:03d}':raise ValueError('Completed epoch receipt identity') + if epoch not in covered:receipts.append((epoch,p)) + backlog=len(receipts)>1;new_records=[] + for epoch,p in receipts: + output=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json' + if output.exists(): + record=json.loads(output.read_text()) + if record['completed_epoch']!=epoch or record['fold']!=fold or record.get('receipt_sha256')!=sha(p):raise ValueError('Existing observation identity') + else: + reason=('TERMINAL_BEFORE_OBSERVATION' if terminal else 'GLOBAL_STOP_BEFORE_OBSERVATION' if stopping else 'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG' if backlog else 'DELAYED_RECEIPT_OVER_5_SECONDS' if time.time()-p.stat().st_mtime>5 else None) + record=epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=p,incomplete_reason=reason) + covered.add(epoch);new_records.append(record) + return new_records diff --git a/tools/phase3_seed1_v2_queue.py b/tools/phase3_seed1_v2_queue.py index 7de984fab..ea44a4960 100644 --- a/tools/phase3_seed1_v2_queue.py +++ b/tools/phase3_seed1_v2_queue.py @@ -12,7 +12,7 @@ from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, save, sha from phase2_home_freeze_io import freeze from s2_recovery_gpu_supervisor import disk_usage, tree_usage -from phase3_seed1_v2_observation import epoch_host_observation +from phase3_seed1_v2_observation import drain_epoch_observations from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus @@ -31,6 +31,7 @@ def run(root): start = time.monotonic() cpu = time.process_time() active, terminal, estimates, last_epoch = {}, {}, {}, {} + observed_epochs = {} pending = list(c['jobs']) requested = [] @@ -62,10 +63,15 @@ def run(root): except psutil.NoSuchProcess: pass + def observe_completed(i, v, terminal=False, stopping=False): + return drain_epoch_observations(root, i, v['job']['gpu_uuid'], + observed_epochs.setdefault(i, set()), terminal=terminal, stopping=stopping) + def collect(i, v): rc = v['process'].poll() if rc is None: return False + observe_completed(i, v, terminal=True, stopping=(root/'global-stop.json').exists()) p = root / 'jobs' / str(i) / 'complete.json' r = read(p) if p.exists() else None observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] @@ -113,7 +119,8 @@ def run(root): try: while active or pending: for i, v in list(active.items()): - collect(i, v) + if not collect(i, v): + observe_completed(i, v, stopping=bool(requested) or (root/'global-stop.json').exists()) for reason in requested: stop(reason) amount = charged() @@ -131,7 +138,6 @@ def run(root): epoch = rows[-1]['epoch'] # Recalculate every epoch: waiting, setup and observed cost stay charged. if epoch > last_epoch.get(i, 0): - epoch_host_observation(root, i, epoch, active[i]['job']['gpu_uuid']) observation = dict(epoch=epoch, charged=active_charges()[i], max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows), max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) @@ -148,7 +154,8 @@ def run(root): if not (root / 'global-stop.json').exists(): dispatch() if (root / 'global-stop.json').exists(): - for v in active.values(): + for i, v in active.items(): + observe_completed(i, v, stopping=True) terminate(v) save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists() else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid) @@ -191,10 +198,15 @@ def run(root): for i, v in list(active.items()): try: if not collect(i, v): + observe_completed(i, v, stopping=True) terminate(v) except Exception: terminate(v) if v['process'].poll() is not None: + try: + observe_completed(i, v, terminal=True, stopping=True) + except Exception as observation_error: + save(root/'jobs'/str(i)/'observation-coverage-error.json', dict(status='OBSERVATION_INCOMPLETE', unix=time.time(), reason=str(observation_error))) terminal[i] = dict(id=i, status='MALFORMED_TERMINAL', gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']) v['log'].close() diff --git a/tools/test_phase3_seed1_v2.py b/tools/test_phase3_seed1_v2.py index 35d1e2714..5914638eb 100644 --- a/tools/test_phase3_seed1_v2.py +++ b/tools/test_phase3_seed1_v2.py @@ -1,4 +1,8 @@ """CPU-only ledger/projection/observation tests for the new stopping rule.""" +import ast +import io +import os +import time import copy import inspect import json @@ -88,4 +92,84 @@ class V2Tests(unittest.TestCase): r=observation.epoch_host_observation(Path(d),1,2,'GPU-A') self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot') +class ObservationCoverageTests(unittest.TestCase): + def queue_fragments(self,root,exited,covered,last_cost,requested=None): + # Execute the actual coordinator's observe/collect and first poll loop. + from phase2_h1_worker_v2 import save,sha + tree=ast.parse((Path(__file__).resolve().parent/'phase3_seed1_v2_queue.py').read_text()) + fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run') + nested=[n for n in fn.body if isinstance(n,ast.FunctionDef) and n.name in ['observe_completed','collect']] + main_try=next(n for n in fn.body if isinstance(n,ast.Try)) + poll_loop=main_try.body[0].body[0] + self.assertIsInstance(poll_loop,ast.For) + v=dict(process=SimpleNamespace(poll=lambda:0 if exited else None),start=time.monotonic(),log=io.StringIO(),job=dict(gpu_uuid='SYNTHETIC-GPU')) + env=dict(root=root,active={0:v},terminal={},observed_epochs={0:set(covered)},last_epoch={0:last_cost},requested=requested or [],c=dict(jobs=[dict(setup_gpu_seconds=60)]),time=time,read=lambda p:json.loads(p.read_text()),save=save,sha=sha,stop=lambda reason:None,drain_epoch_observations=observation.drain_epoch_observations) + code=compile(ast.fix_missing_locations(ast.Module(body=nested+[poll_loop],type_ignores=[])),'actual_queue_poll','exec') + exec(code,env) + return env + + def receipts(self,root,epochs): + from phase2_h1_worker_v2 import save + d=root/'jobs/0';(d/'worker/history').mkdir(parents=True) + save(d/'assignment.json',{}) + save(d/'complete.json',dict(status='PASS',monitor_errors=[],worker=dict(monitor_errors=0),gpu_seconds_charged=1000)) + for e in epochs:save(d/'worker/history'/f'{e:03d}.json',dict(epoch=e,train_epoch_seconds=1)) + return d + + def test_terminal_before_next_poll_keeps_epoch200(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);job=self.receipts(root,[199,200]) + with patch.object(observation.subprocess,'run') as query: + env=self.queue_fragments(root,True,range(1,200),199) + query.assert_not_called() + record=json.loads((job/'epoch-host-observations/200.json').read_text()) + self.assertEqual(record['status'],'OBSERVATION_INCOMPLETE') + self.assertEqual(record['incomplete_reason'],'TERMINAL_BEFORE_OBSERVATION') + self.assertIsNone(record['snapshot_started_unix']);self.assertEqual(record['gpu_processes'],[]) + self.assertEqual(env['active'],{});self.assertEqual(env['observed_epochs'][0],set(range(1,201)));self.assertEqual(env['last_epoch'][0],199) + + def test_two_unseen_receipts_each_covered_no_historical_snapshot(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);job=self.receipts(root,[199,200]) + with patch.object(observation.subprocess,'run') as query: + env=self.queue_fragments(root,False,range(1,199),198) + self.assertEqual(observation.drain_epoch_observations(root,0,'SYNTHETIC-GPU',env['observed_epochs'][0]),[]) + query.assert_not_called() + for epoch in [199,200]: + record=json.loads((job/'epoch-host-observations'/f'{epoch:03d}.json').read_text()) + self.assertEqual(record['incomplete_reason'],'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG');self.assertIsNone(record['snapshot_started_unix']) + self.assertIn(0,env['active']);self.assertEqual(env['last_epoch'][0],198);self.assertEqual(env['observed_epochs'][0],set(range(1,201))) + + def test_global_stop_poll_covers_completed_receipt(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);job=self.receipts(root,[12]);(root/'global-stop.json').write_text('{}') + with patch.object(observation.subprocess,'run') as query: + env=self.queue_fragments(root,False,range(1,12),11) + query.assert_not_called() + record=json.loads((job/'epoch-host-observations/012.json').read_text()) + self.assertEqual(record['incomplete_reason'],'GLOBAL_STOP_BEFORE_OBSERVATION');self.assertIn(12,env['observed_epochs'][0]) + + def test_delayed_single_receipt_is_incomplete_without_query(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);job=self.receipts(root,[1]);p=job/'worker/history/001.json';old=time.time()-20;os.utime(p,(old,old)) + with patch.object(observation.subprocess,'run') as query: + records=observation.drain_epoch_observations(root,0,'GPU-A',set()) + query.assert_not_called();self.assertEqual(records[0]['incomplete_reason'],'DELAYED_RECEIPT_OVER_5_SECONDS');self.assertGreaterEqual(records[0]['receipt_age_seconds'],20) + + def test_fresh_single_receipt_has_real_snapshot_times_and_hash(self): + from phase2_h1_worker_v2 import sha + with tempfile.TemporaryDirectory() as d: + root=Path(d);job=self.receipts(root,[1]);start=time.time() + own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d])) + with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='GPU-A, 10, python, 800\n')): + records=observation.drain_epoch_observations(root,0,'GPU-A',set()) + r=records[0];self.assertEqual(r['status'],'OBSERVED');self.assertGreaterEqual(r['snapshot_started_unix'],start);self.assertGreaterEqual(r['snapshot_finished_unix'],r['snapshot_started_unix']);self.assertFalse(r['historical_gpu_occupancy_observed']);self.assertEqual(r['receipt_sha256'],sha(job/'worker/history/001.json')) + + def test_worker_disappearing_between_poll_and_snapshot_is_incomplete(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);self.receipts(root,[200]) + with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='')): + records=observation.drain_epoch_observations(root,0,'GPU-A',set()) + self.assertEqual(records[0]['status'],'OBSERVATION_INCOMPLETE');self.assertEqual(records[0]['errors'][0]['stage'],'epoch_worker_visibility');self.assertIsNotNone(records[0]['snapshot_started_unix']) + if __name__=='__main__':unittest.main()