Apply model-only 60 CPU hour amendment with sole waiter replacement evidence

This commit is contained in:
Frank14f committed 2026-11-16 00:27:49 +08:00
1 parent 97033fcfa5
commit 1159fdcc30
9 files changed
+418

No files matched your search

@@ -0,0 +1,52 @@
{
"unix": 1794760016.3214407,
"session": 97915,
"pid": 482008,
"cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"-u",
"tools/run_gre24_models.py",
"artifacts/gre24-reproduction-2026-11-15/v5-models-60h",
"artifacts/gre24-reproduction-2026-11-15/v5-models-60h/model-plan.json"
],
"create_time": 1794759994.37,
"status": "sleeping",
"children": [],
"started": {
"pid": 482008,
"unix": 1794759995.5108879,
"plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"stage": "WAIT_PREPARATION"
},
"progress": {
"stage": "WAIT_PREPARATION",
"unix": 1794760015.5282362
},
"new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"loaded_plan_matches_started": true,
"changed_plan_fields": [
"total_worker_cpu_seconds"
],
"effective_total_worker_cpu_seconds": 216000,
"effective_total_gpu_seconds": 7200,
"initial_gpu_seconds": 10.185812931973487,
"old_waiter_alive": false,
"old_tool_session_exit_code": 143,
"old_plan_unchanged": true,
"preparation_pid": 479550,
"preparation_alive": true,
"preparation_contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248",
"preparation_limits": {
"workers": 8,
"total_worker_cpu_seconds": 144000,
"per_part_cpu_seconds": 1800,
"per_part_wall_seconds": 1800,
"per_part_rss_bytes": 4294967296,
"preparation_initial_cpu_reserve_seconds": 82.684837,
"total_disk_bytes": 60000000000,
"total_gpu_seconds": 7200
},
"preparation_source_pins_match": true,
"model_source_pins_match": true,
"training_started": false
}
@@ -0,0 +1,29 @@
{
"authorizing_ruling": "1794759849.111939",
"unix": 1794759924.8378572,
"old_root": "artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"new_root": "artifacts/gre24-reproduction-2026-11-15/v5-models-60h",
"old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4",
"new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"changed_fields": {
"total_worker_cpu_seconds": {
"old": 144000,
"new": 216000
}
},
"unchanged_gpu_seconds": 7200,
"initial_gpu_seconds": 10.185812931973487,
"source_pins_unchanged": true,
"preparation_contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248",
"preparation_coordinator_pid": 479550,
"preparation_limit_unchanged": true,
"linked_preparation_inputs": {
"contract.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/contract.json",
"splits.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/splits.json",
"queue-started.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/queue-started.json",
"complete.json": "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/v5-protocol/complete.json"
},
"historical_plan_note": "Only numeric total CPU field changes. The inherited descriptive limits_notes text saying40h is superseded by this recorded amendment.",
"replacement_policy": "Retire only idle original model waiter after frozen process identity/no-children/no-fit check; preserve original markers/log/plan. No preparation restart or scientific retry.",
"launch_status": "PREPARED_NOT_STARTED"
}
@@ -0,0 +1,6 @@
{
"pid": 482008,
"unix": 1794759995.5108879,
"plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"stage": "WAIT_PREPARATION"
}
@@ -0,0 +1,63 @@
{
"version": "Gre24-v5-model-chain-v1",
"dataset_version": "v5",
"protocol": "5 independent random 312test/312train geometry splits; v5 data deviation",
"authorizing_rulings": [
"1794755127.457709",
"1794755474.778649",
"1794755594.435869",
"1794756165.827889",
"1794756271.768399"
],
"code_sha256": {
"tools/run_gre24_models.py": "103ecb6dc8a40481f72da02a02e0a41069a9a747ad0b7af4418c6ed06f0dcbc9",
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
"tools/gre24_score.py": "dbb11b57b7b7b2178ac11fd7243c934adb899d666a4c717dfb6a51ee1c65c705",
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa",
"tools/gre24_prepare_v2.py": "7637e55009005f6a8c77114159d7ac0d3c024c7add3e320afc486927537b78bc",
"tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919"
},
"total_worker_cpu_seconds": 216000,
"total_gpu_seconds": 7200,
"initial_gpu_seconds": 10.185812931973487,
"postfit_cpu_reserve_seconds": 900,
"xgb_per_fit_cpu_seconds": 7200,
"fnn_per_fit_cpu_seconds": 14400,
"xgb_threads": 8,
"fnn_cpu_threads": 2,
"gpu_reserved_byte_cap": 16106127360,
"ordering": "All five GBRT, then five FNN, freeze every terminal/checkpoint, CPU prediction freeze, CPU scoring",
"FNN": {
"hidden_layers": 4,
"width": 100,
"activation": "ReLU",
"optimizer": "Adam",
"learning_rate": 0.001,
"batch_size": 1024,
"maximum_epochs": 30,
"patience": 5,
"restore_best": true,
"AMP": false,
"input_standardization": "none beyond Eq12",
"validation": "32 fixed training geometries per split; no replacement after prep failure"
},
"GBRT": {
"library": "xgboost2.1.4",
"trees": 100,
"max_depth": 8,
"learning_rate": 0.1,
"objective": "reg:squarederror",
"other_parameters": "library defaults, saved per fit"
},
"limits_notes": "No retry or replacement; CPU preparation+oldcost+test reserve+all fits+postfit share40h; waits/setup/failures in FNN stage count toward shared2GPUh",
"acceptance_receipts": {
"artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json": "008c40416c992e27b7111a7a5580ef151be29849fa5f275f70e7cc22ca196397",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json": "e2d11ca8901285999e7e71b2f94f6f14a586252cc5797881a5eede72fdd716a3",
"artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json": "90958f1e44239c9c776a7c297ba2b848193e208ab74881ba63c62bb04328bb15"
},
"timestamp": 1794757401.999023
}
@@ -0,0 +1,40 @@
{
"unix": 1794759981.3739972,
"ruling": "1794759849.111939",
"pid": 481142,
"session": 72377,
"cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"-u",
"tools/run_gre24_models.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json"
],
"process_create_time": 1794758119.95,
"frozen_status": "stopped",
"children": [],
"training_started": false,
"preparation_complete": false,
"progress": {
"stage": "WAIT_PREPARATION",
"unix": 1794759976.0861633
},
"old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4",
"new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"original_files_sha256": {
"model-chain-started.json": "18563d3356460e53f32aa7084ce3ac1e424b7fd154cabe850b872813b0a1bdfb",
"model-chain-progress.json": "bcaeed4b790315d269066db04018511242ceafcf5e35d02f2f958fb93244ea38",
"model-chain.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"model-plan.json": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4"
},
"reason": "Authorized idle waiter retirement for model-only CPU budget amendment; no scientific job interrupted",
"exit_observed_unix": 1794759981.4006045,
"wait_status": null,
"original_pid_alive": false,
"preparation_still_running": true,
"preparation_cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/run_gre24_protocol_preparation.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol"
]
}
@@ -0,0 +1,6 @@
{
"status": "PRECHECK_ABORTED_NO_TERMINATION",
"reason": "SIGSTOP status read immediately before kernel reported stopped; finally resumed original waiter",
"source": "previous command AssertionError line10",
"unix": 1794759981.3049247
}
@@ -0,0 +1,31 @@
{
"unix": 1794759981.3739972,
"ruling": "1794759849.111939",
"pid": 481142,
"session": 72377,
"cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"-u",
"tools/run_gre24_models.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json"
],
"process_create_time": 1794758119.95,
"frozen_status": "stopped",
"children": [],
"training_started": false,
"preparation_complete": false,
"progress": {
"stage": "WAIT_PREPARATION",
"unix": 1794759976.0861633
},
"old_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4",
"new_plan_sha256": "7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a",
"original_files_sha256": {
"model-chain-started.json": "18563d3356460e53f32aa7084ce3ac1e424b7fd154cabe850b872813b0a1bdfb",
"model-chain-progress.json": "bcaeed4b790315d269066db04018511242ceafcf5e35d02f2f958fb93244ea38",
"model-chain.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"model-plan.json": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4"
},
"reason": "Authorized idle waiter retirement for model-only CPU budget amendment; no scientific job interrupted"
}
@@ -0,0 +1,170 @@
{
"unix": 1794759802.9313865,
"time_utc8": "2026-11-16T00:23:22.931399+08:00",
"processes": {
"preparation": {
"pid": 479550,
"status": "sleeping",
"cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/run_gre24_protocol_preparation.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol"
],
"identity_matches": true,
"create_time": 1794756410.19
},
"chain": {
"pid": 481142,
"status": "sleeping",
"cmdline": [
"artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"-u",
"tools/run_gre24_models.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json"
],
"identity_matches": true,
"create_time": 1794758119.95
}
},
"progress": {
"completed": 112,
"attempted": 120,
"total": 624,
"active": {
"1": 74,
"5": 75,
"4": 112,
"3": 113,
"6": 116,
"0": 117,
"2": 118,
"7": 119
},
"settled_worker_cpu_seconds": 20809.881566000007,
"reserved_active_worker_cpu_seconds": 14416,
"stop_reason": null,
"first_part_checked": true,
"elapsed_seconds": 3391.4999872120097
},
"terminal_counts": {
"PASS": 111,
"STOPPED_OR_FAILED": 1
},
"failures": [
{
"case": "00000033_ad34a3f60c4a4caa99646600_step_013",
"status": "STOPPED_OR_FAILED",
"reason": "wall_limit",
"cpu_seconds": 1704.651106,
"monitor_errors": 0
}
],
"terminal_receipt_cpu_seconds": 20727.196729000003,
"live_workers": [
{
"pid": 481817,
"owner": "preparation",
"cpu_seconds": 5.53,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"119"
]
},
{
"pid": 481815,
"owner": "preparation",
"cpu_seconds": 6.5,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"118"
]
},
{
"pid": 481806,
"owner": "preparation",
"cpu_seconds": 23.88,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"117"
]
},
{
"pid": 481765,
"owner": "preparation",
"cpu_seconds": 94.01,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"116"
]
},
{
"pid": 481735,
"owner": "preparation",
"cpu_seconds": 252.0,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"113"
]
},
{
"pid": 481730,
"owner": "preparation",
"cpu_seconds": 265.65000000000003,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"112"
]
},
{
"pid": 481282,
"owner": "preparation",
"cpu_seconds": 1509.53,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"75"
]
},
{
"pid": 481163,
"owner": "preparation",
"cpu_seconds": 1568.1799999999998,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"artifacts/gre24-reproduction-2026-11-15/v5-protocol",
"74"
]
}
],
"live_worker_cpu_seconds": 3725.2799999999997,
"remaining_cpu_seconds_sampled": 119464.83843399999,
"preparation_source_mismatches": [],
"model_source_mismatches": [],
"contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248",
"model_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4",
"split_pin_matches": true,
"training_started_files": [],
"initial_gpu_seconds": 10.185812931973487,
"chain_progress": {
"stage": "WAIT_PREPARATION",
"unix": 1794759795.691558
},
"complete": null,
"reproduction_complete": null,
"chain_log_tail": ""
}
+21
View File
@@ -0,0 +1,21 @@
# Gre24 模型接续 CPU 预算修订
Claude 原串 `1794759849.111939`(00:24:09,UTC+8)已实读:本次复现总 worker CPU 上限从 144000 提高到 **216000 秒(60 小时)**,GPU 仍为 **7200 秒**。仅模型链采用新总帽;原准备进程保持 144000 秒,不重启。单件 1800 秒 / 4 GiB、失败保留、无替补/重试均不变。
旧等待器在启动时一次读入计划,修改磁盘文件无法使其内存中的预算生效。因此仅退役空闲模型等待器,保留原根目录、计划、started、progress 与日志;相同科学源码从独立根目录启动唯一替代等待链。没有重启准备或重试科学任务。
## 固定计划与实际生效
- 原准备根:`artifacts/gre24-reproduction-2026-11-15/v5-protocol`,session **18323** / PID **479550**,合同 SHA `9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248`,源码与限制均不变。
- 原模型等待:session **72377** / PID **481142** 已主动退役,工具回执 exit **143**。先暂停并验证身份、无 children、无 fits、无 training-contract、准备尚未 complete,再发 SIGTERM;退役后 PID 不存在。
- 唯一新模型等待:session **97915** / PID **482008**,根 `artifacts/gre24-reproduction-2026-11-15/v5-models-60h`,00:26:56 核到存活、children=[]、`WAIT_PREPARATION`,未开始训练。
- 原 model-plan SHA `4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4` 保留。新 SHA **`7f6cfbd1c4ebee9f61976cf45112fb0ef70f69d47cdb559ae86f0b643c41118a`** 与新进程 started 记录一致。机器比较仅 `total_worker_cpu_seconds` 字段变化(144000→216000);源码 pins 完全相同。
- 新根以链接读取同一准备合同、划分、queue-started 与未来 complete,输出模型/预测/评分在新根;没有复制或重算准备结果。完整链接目标记录于 `budget-amendment.json`。继承计划的历史说明文字仍提到40h,由此修订记录明确取代,执行使用数值字段216000。
历史/合成 CPU 仍由原准备完整账本带入,只计一次;GPU 历史 **10.185812931973487 秒**亦不变。624 完整终态且无全局停止、费用完整后才训练;全部模型终态/checkpoint 冻结后预测,全部预测冻结后评分。00000033 的 wall_limit 与后续任何失败保留,不补件。
## 工程证据与限制
新根保存 `retirement-precheck.json`、`old-waiter-retired.json`、`actual-start-audit.json` 及原始 started。第一次暂停核查在内核尚未报告 stopped 时立即检查失败,finally 恢复原等待器,未终止;保留 `retirement-check-attempt-01.json`。第二次在限定等待确认 stopped 后完成核查和退役。该工程检查没有触发任何拟合、GPU 工作或科学重试。
本次只修订合同与接续记录,无科学源码修改、参数变化或新增实验;已有合成验收仍适用。后续轮询应检查新等待 PID482008/根目录,不能重启已退役481142。