diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/actual-start-audit.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/actual-start-audit.json new file mode 100644 index 000000000..a3f7d3f19 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/actual-start-audit.json @@ -0,0 +1,52 @@ +{ + "unix": 1794760016.3214407, + "session": 97915, + "pid": 482008, + "cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_gre24_models.py", + "artifacts/gre24-reproduction-2026-11-15/v5-models-60h", + "artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-plan.json" + ], + "create_time": 1794759994.37, + "status": "sleeping", + "children": [], + "started": { + "pid": 482008, + "unix": 1794759995.5108879, + "plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "stage": "WAIT_PREPARATION" + }, + "progress": { + "stage": "WAIT_PREPARATION", + "unix": 1794760015.5282362 + }, + "new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "loaded_plan_matches_started": true, + "changed_plan_fields": [ + "total_worker_cpu_seconds" + ], + "effective_total_worker_cpu_seconds": 216000, + "effective_total_gpu_seconds": 7200, + "initial_gpu_seconds": 10.185812931973487, + "old_waiter_alive": false, + "old_tool_session_exit_code": 143, + "old_plan_unchanged": true, + "preparation_pid": 479550, + "preparation_alive": true, + "preparation_contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248", + "preparation_limits": { + "workers": 8, + "total_worker_cpu_seconds": 144000, + "per_part_cpu_seconds": 1800, + "per_part_wall_seconds": 1800, + "per_part_rss_bytes": 4294967296, + "preparation_initial_cpu_reserve_seconds": 82.684837, + "total_disk_bytes": 60000000000, + "total_gpu_seconds": 7200 + }, + "preparation_source_pins_match": true, + "model_source_pins_match": true, + "training_started": false +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/budget-amendment.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/budget-amendment.json new file mode 100644 index 000000000..d2b6b3c19 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/budget-amendment.json @@ -0,0 +1,29 @@ +{ + "authorizing_ruling": "1794759849.111939", + "unix": 1794759924.8378572, + "old_root": "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "new_root": "artifacts/gre24-reproduction-2026-11-15/v5-models-60h", + "old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4", + "new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "changed_fields": { + "total_worker_cpu_seconds": { + "old": 144000, + "new": 216000 + } + }, + "unchanged_gpu_seconds": 7200, + "initial_gpu_seconds": 10.185812931973487, + "source_pins_unchanged": true, + "preparation_contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248", + "preparation_coordinator_pid": 479550, + "preparation_limit_unchanged": true, + "linked_preparation_inputs": { + "contract.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/contract.json", + "splits.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/splits.json", + "queue-started.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/queue-started.json", + "complete.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/complete.json" + }, + "historical_plan_note": "Only numeric total CPU field changes. The inherited descriptive limits_notes text saying40h is superseded by this recorded amendment.", + "replacement_policy": "Retire only idle original model waiter after frozen process identity/no-children/no-fit check; preserve original markers/log/plan. No preparation restart or scientific retry.", + "launch_status": "PREPARED_NOT_STARTED" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-chain-started.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-chain-started.json new file mode 100644 index 000000000..7489bd489 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-chain-started.json @@ -0,0 +1,6 @@ +{ + "pid": 482008, + "unix": 1794759995.5108879, + "plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "stage": "WAIT_PREPARATION" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-plan.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-plan.json new file mode 100644 index 000000000..d4e9fb1c9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-plan.json @@ -0,0 +1,63 @@ +{ + "version": "Gre24-v5-model-chain-v1", + "dataset_version": "v5", + "protocol": "5 independent random 312test/312train geometry splits; v5 data deviation", + "authorizing_rulings": [ + "1794755127.457709", + "1794755474.778649", + "1794755594.435869", + "1794756165.827889", + "1794756271.768399" + ], + "code_sha256": { + "tools/run_gre24_models.py": "103ecb6dc8a40481f72da02a02e0a41069a9a747ad0b7af4418c6ed06f0dcbc9", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "dbb11b57b7b7b2178ac11fd7243c934adb899d666a4c717dfb6a51ee1c65c705", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa", + "tools/gre24_prepare_v2.py": "7637e55009005f6a8c77114159d7ac0d3c024c7add3e320afc486927537b78bc", + "tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919" + }, + "total_worker_cpu_seconds": 216000, + "total_gpu_seconds": 7200, + "initial_gpu_seconds": 10.185812931973487, + "postfit_cpu_reserve_seconds": 900, + "xgb_per_fit_cpu_seconds": 7200, + "fnn_per_fit_cpu_seconds": 14400, + "xgb_threads": 8, + "fnn_cpu_threads": 2, + "gpu_reserved_byte_cap": 16106127360, + "ordering": "All five GBRT, then five FNN, freeze every terminal/checkpoint, CPU prediction freeze, CPU scoring", + "FNN": { + "hidden_layers": 4, + "width": 100, + "activation": "ReLU", + "optimizer": "Adam", + "learning_rate": 0.001, + "batch_size": 1024, + "maximum_epochs": 30, + "patience": 5, + "restore_best": true, + "AMP": false, + "input_standardization": "none beyond Eq12", + "validation": "32 fixed training geometries per split; no replacement after prep failure" + }, + "GBRT": { + "library": "xgboost2.1.4", + "trees": 100, + "max_depth": 8, + "learning_rate": 0.1, + "objective": "reg:squarederror", + "other_parameters": "library defaults, saved per fit" + }, + "limits_notes": "No retry or replacement; CPU preparation+oldcost+test reserve+all fits+postfit share40h; waits/setup/failures in FNN stage count toward shared2GPUh", + "acceptance_receipts": { + "artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json": "008c40416c992e27b7111a7a5580ef151be29849fa5f275f70e7cc22ca196397", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json": "e2d11ca8901285999e7e71b2f94f6f14a586252cc5797881a5eede72fdd716a3", + "artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json": "90958f1e44239c9c776a7c297ba2b848193e208ab74881ba63c62bb04328bb15" + }, + "timestamp": 1794757401.999023 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/old-waiter-retired.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/old-waiter-retired.json new file mode 100644 index 000000000..72f713125 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/old-waiter-retired.json @@ -0,0 +1,40 @@ +{ + "unix": 1794759981.3739972, + "ruling": "1794759849.111939", + "pid": 481142, + "session": 72377, + "cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_gre24_models.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json" + ], + "process_create_time": 1794758119.95, + "frozen_status": "stopped", + "children": [], + "training_started": false, + "preparation_complete": false, + "progress": { + "stage": "WAIT_PREPARATION", + "unix": 1794759976.0861633 + }, + "old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4", + "new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "original_files_sha256": { + "model-chain-started.json": "18563d3356460e53f32aa7084ce3ac1e424b7fd154cabe850b872813b0a1bdfb", + "model-chain-progress.json": "bcaeed4b790315d269066db04018511242ceafcf5e35d02f2f958fb93244ea38", + "model-chain.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "model-plan.json": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4" + }, + "reason": "Authorized idle waiter retirement for model-only CPU budget amendment; no scientific job interrupted", + "exit_observed_unix": 1794759981.4006045, + "wait_status": null, + "original_pid_alive": false, + "preparation_still_running": true, + "preparation_cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/run_gre24_protocol_preparation.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol" + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-check-attempt-01.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-check-attempt-01.json new file mode 100644 index 000000000..53b9bf215 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-check-attempt-01.json @@ -0,0 +1,6 @@ +{ + "status": "PRECHECK_ABORTED_NO_TERMINATION", + "reason": "SIGSTOP status read immediately before kernel reported stopped; finally resumed original waiter", + "source": "previous command AssertionError line10", + "unix": 1794759981.3049247 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-precheck.json b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-precheck.json new file mode 100644 index 000000000..8d9ddb71c --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-models-60h/retirement-precheck.json @@ -0,0 +1,31 @@ +{ + "unix": 1794759981.3739972, + "ruling": "1794759849.111939", + "pid": 481142, + "session": 72377, + "cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_gre24_models.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json" + ], + "process_create_time": 1794758119.95, + "frozen_status": "stopped", + "children": [], + "training_started": false, + "preparation_complete": false, + "progress": { + "stage": "WAIT_PREPARATION", + "unix": 1794759976.0861633 + }, + "old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4", + "new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a", + "original_files_sha256": { + "model-chain-started.json": "18563d3356460e53f32aa7084ce3ac1e424b7fd154cabe850b872813b0a1bdfb", + "model-chain-progress.json": "bcaeed4b790315d269066db04018511242ceafcf5e35d02f2f958fb93244ea38", + "model-chain.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "model-plan.json": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4" + }, + "reason": "Authorized idle waiter retirement for model-only CPU budget amendment; no scientific job interrupted" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-protocol/readonly-status-1794759802.json b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/readonly-status-1794759802.json new file mode 100644 index 000000000..c35629e4f --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/readonly-status-1794759802.json @@ -0,0 +1,170 @@ +{ + "unix": 1794759802.9313865, + "time_utc8": "2026-11-16T00:23:22.931399+08:00", + "processes": { + "preparation": { + "pid": 479550, + "status": "sleeping", + "cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/run_gre24_protocol_preparation.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol" + ], + "identity_matches": true, + "create_time": 1794756410.19 + }, + "chain": { + "pid": 481142, + "status": "sleeping", + "cmdline": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_gre24_models.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json" + ], + "identity_matches": true, + "create_time": 1794758119.95 + } + }, + "progress": { + "completed": 112, + "attempted": 120, + "total": 624, + "active": { + "1": 74, + "5": 75, + "4": 112, + "3": 113, + "6": 116, + "0": 117, + "2": 118, + "7": 119 + }, + "settled_worker_cpu_seconds": 20809.881566000007, + "reserved_active_worker_cpu_seconds": 14416, + "stop_reason": null, + "first_part_checked": true, + "elapsed_seconds": 3391.4999872120097 + }, + "terminal_counts": { + "PASS": 111, + "STOPPED_OR_FAILED": 1 + }, + "failures": [ + { + "case": "00000033_ad34a3f60c4a4caa99646600_step_013", + "status": "STOPPED_OR_FAILED", + "reason": "wall_limit", + "cpu_seconds": 1704.651106, + "monitor_errors": 0 + } + ], + "terminal_receipt_cpu_seconds": 20727.196729000003, + "live_workers": [ + { + "pid": 481817, + "owner": "preparation", + "cpu_seconds": 5.53, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "119" + ] + }, + { + "pid": 481815, + "owner": "preparation", + "cpu_seconds": 6.5, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "118" + ] + }, + { + "pid": 481806, + "owner": "preparation", + "cpu_seconds": 23.88, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "117" + ] + }, + { + "pid": 481765, + "owner": "preparation", + "cpu_seconds": 94.01, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "116" + ] + }, + { + "pid": 481735, + "owner": "preparation", + "cpu_seconds": 252.0, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "113" + ] + }, + { + "pid": 481730, + "owner": "preparation", + "cpu_seconds": 265.65000000000003, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "112" + ] + }, + { + "pid": 481282, + "owner": "preparation", + "cpu_seconds": 1509.53, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "75" + ] + }, + { + "pid": 481163, + "owner": "preparation", + "cpu_seconds": 1568.1799999999998, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol", + "74" + ] + } + ], + "live_worker_cpu_seconds": 3725.2799999999997, + "remaining_cpu_seconds_sampled": 119464.83843399999, + "preparation_source_mismatches": [], + "model_source_mismatches": [], + "contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248", + "model_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4", + "split_pin_matches": true, + "training_started_files": [], + "initial_gpu_seconds": 10.185812931973487, + "chain_progress": { + "stage": "WAIT_PREPARATION", + "unix": 1794759795.691558 + }, + "complete": null, + "reproduction_complete": null, + "chain_log_tail": "" +} diff --git a/docs/gre24-budget-amendment-2026-11-16.md b/docs/gre24-budget-amendment-2026-11-16.md new file mode 100644 index 000000000..325821a10 --- /dev/null +++ b/docs/gre24-budget-amendment-2026-11-16.md @@ -0,0 +1,21 @@ +# Gre24 模型接续 CPU 预算修订 + +Claude 原串 `1794759849.111939`(00:24:09,UTC+8)已实读:本次复现总 worker CPU 上限从 144000 提高到 **216000 秒(60 小时)**,GPU 仍为 **7200 秒**。仅模型链采用新总帽;原准备进程保持 144000 秒,不重启。单件 1800 秒 / 4 GiB、失败保留、无替补/重试均不变。 + +旧等待器在启动时一次读入计划,修改磁盘文件无法使其内存中的预算生效。因此仅退役空闲模型等待器,保留原根目录、计划、started、progress 与日志;相同科学源码从独立根目录启动唯一替代等待链。没有重启准备或重试科学任务。 + +## 固定计划与实际生效 + +- 原准备根:`artifacts/gre24-reproduction-2026-11-15/v5-protocol`,session **18323** / PID **479550**,合同 SHA `9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248`,源码与限制均不变。 +- 原模型等待:session **72377** / PID **481142** 已主动退役,工具回执 exit **143**。先暂停并验证身份、无 children、无 fits、无 training-contract、准备尚未 complete,再发 SIGTERM;退役后 PID 不存在。 +- 唯一新模型等待:session **97915** / PID **482008**,根 `artifacts/gre24-reproduction-2026-11-15/v5-models-60h`,00:26:56 核到存活、children=[]、`WAIT_PREPARATION`,未开始训练。 +- 原 model-plan SHA `4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4` 保留。新 SHA **`7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a`** 与新进程 started 记录一致。机器比较仅 `total_worker_cpu_seconds` 字段变化(144000→216000);源码 pins 完全相同。 +- 新根以链接读取同一准备合同、划分、queue-started 与未来 complete,输出模型/预测/评分在新根;没有复制或重算准备结果。完整链接目标记录于 `budget-amendment.json`。继承计划的历史说明文字仍提到40h,由此修订记录明确取代,执行使用数值字段216000。 + +历史/合成 CPU 仍由原准备完整账本带入,只计一次;GPU 历史 **10.185812931973487 秒**亦不变。624 完整终态且无全局停止、费用完整后才训练;全部模型终态/checkpoint 冻结后预测,全部预测冻结后评分。00000033 的 wall_limit 与后续任何失败保留,不补件。 + +## 工程证据与限制 + +新根保存 `retirement-precheck.json`、`old-waiter-retired.json`、`actual-start-audit.json` 及原始 started。第一次暂停核查在内核尚未报告 stopped 时立即检查失败,finally 恢复原等待器,未终止;保留 `retirement-check-attempt-01.json`。第二次在限定等待确认 stopped 后完成核查和退役。该工程检查没有触发任何拟合、GPU 工作或科学重试。 + +本次只修订合同与接续记录,无科学源码修改、参数变化或新增实验;已有合成验收仍适用。后续轮询应检查新等待 PID482008/根目录,不能重启已退役481142。