Record independent research parallelism and withdrawn unstarted throughput draft

This commit is contained in:
MeltAstra committed 2026-11-18 21:03:33 +08:00
1 parent ff0033d733
commit 5b6a953fc5
10 files changed
+1812

No files matched your search

@@ -0,0 +1,427 @@
[
{
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"code_pins": 51,
"pins": "PASS",
"budget": {
"gpu_cap": 158717.68659210298,
"train_gpu_cap": 151517.68659210298,
"postfit_gpu_reserve": 7200
},
"jobs": [
{
"id": 0,
"gpu_uuid": null,
"worker_cpus": [
0,
1
],
"guard_cpus": [
10
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 1,
"gpu_uuid": null,
"worker_cpus": [
2,
3
],
"guard_cpus": [
11
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 2,
"gpu_uuid": null,
"worker_cpus": [
4,
5
],
"guard_cpus": [
12
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 3,
"gpu_uuid": null,
"worker_cpus": [
6,
7
],
"guard_cpus": [
13
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 4,
"gpu_uuid": null,
"worker_cpus": [
8,
9
],
"guard_cpus": [
14
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
}
],
"receipts": {
"live-progress.json": {
"stage": "SEED1_TRAINING",
"unix": 1795006694.1644843,
"active": [
{
"id": 0,
"guard_pid": 1476627
},
{
"id": 1,
"guard_pid": 1476628
},
{
"id": 2,
"guard_pid": 1598086
}
],
"terminal": [],
"pending": [
3,
4
],
"charged_training_gpu_seconds": 36065.127459958894,
"postfit_gpu_reserve": 7200
},
"latest-cost-projection.json": {
"status": "PASS",
"rows": [
{
"id": 0,
"projected_gpu_seconds": 20537.820046406356,
"actual_charged_gpu_seconds": 14611.722367653972,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 1,
"projected_gpu_seconds": 29538.486472418066,
"actual_charged_gpu_seconds": 14611.717376481974,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 2,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 6834.808777031023,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 3,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 0,
"actual_within_runaway": true,
"basis": "pending_median_reserve"
},
{
"id": 4,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 0,
"actual_within_runaway": true,
"basis": "pending_median_reserve"
}
],
"projected_gpu_seconds": 134233.61350179132,
"phase3_prior_gpu_once": 80903.62940728385,
"projected_phase3_gpu_seconds": 215137.24290907517,
"postfit_reserve": 7200,
"pending_reserve_gpu_seconds": 25652.435660988966,
"actual_runaway_gpu_seconds": 40000,
"total_projection_within_cap": true,
"cell": {
"id": 2,
"projected_gpu_seconds": 25652.435660988966,
"validation_seconds_bound": 9.668979812064208,
"actual_charged_gpu_seconds": 6834.808727527969,
"actual_within_runaway": true,
"projected_above_runaway": false
},
"contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981",
"unix": 1795006691.8720207
}
},
"workers": [
{
"job": "0",
"progress": {
"fold": 0,
"stage": "TRAIN",
"epoch": 155,
"optimizer_steps": 49441,
"wall_seconds": 14543.364778421004,
"cpu_seconds": 14504.733303124001,
"case": "00000072_767e4372b5f94a88a7a17d90_step_013",
"epoch_parts": 161,
"epoch_nodes": 4444056,
"lr": 0.00014081666421679985,
"loss": 2.4992827093228698e-05,
"gradient_norm_before_clip": 0.000830289616715163,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1388314624
},
"completed_epochs": 154,
"recent10_epoch_seconds": [
91.79020690300968,
91.0233701330144,
91.02645130397286,
91.52249923499767,
95.11965373891871,
93.42180484591518,
89.95967863500118,
92.87886077200528,
91.14694070699625,
93.60838667897042
],
"median_recent10": 91.65635306900367,
"peak_allocated": 1334860288,
"peak_reserved": 1388314624
},
{
"job": "1",
"progress": {
"fold": 1,
"stage": "TRAIN",
"epoch": 119,
"optimizer_steps": 44474,
"wall_seconds": 14543.898690835922,
"cpu_seconds": 14503.983612828,
"case": "00000790_a25fb37392a44447b9684f16_step_015",
"epoch_parts": 342,
"epoch_nodes": 8888977,
"lr": 0.00039143515957685817,
"loss": 0.0005089675541967154,
"gradient_norm_before_clip": 0.01804092526435852,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1390411776
},
"completed_epochs": 118,
"recent10_epoch_seconds": [
104.03197531797923,
104.27232543297578,
102.27148887095973,
104.10447630204726,
103.55956940201577,
107.4991078999592,
105.82303702400532,
102.22429705096874,
106.02757873607334,
104.48560117301531
],
"median_recent10": 104.18840086751152,
"peak_allocated": 1334860288,
"peak_reserved": 1390411776
},
{
"job": "2",
"progress": {
"fold": 2,
"stage": "TRAIN",
"epoch": 65,
"optimizer_steps": 23435,
"wall_seconds": 6767.362545815995,
"cpu_seconds": 6766.308971986,
"case": "00001368_2e314c6e080b40fcb6a6db96_step_011",
"epoch_parts": 11,
"epoch_nodes": 263923,
"lr": 0.0008090352927813857,
"loss": 0.00021560114691965282,
"gradient_norm_before_clip": 0.030973436310887337,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"completed_epochs": 64,
"recent10_epoch_seconds": [
103.85552064399235,
102.38861383905169,
103.54317934100982,
102.5205716479104,
107.35395550006069,
106.42893402301706,
102.09753456991166,
105.13476634700783,
104.87566378701013,
104.03085032501258
],
"median_recent10": 103.94318548450246,
"peak_allocated": 1461572608,
"peak_reserved": 1516240896
}
]
},
{
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
"code_pins": 27,
"pins": "PASS",
"budget": {
"gpu_cap": 162000,
"train_gpu_cap": 106856.05280544702,
"postfit_gpu_reserve": 7200
},
"jobs": [
{
"id": 0,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 1,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"worker_cpus": [
10
],
"guard_cpus": [
29
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 2,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 3,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"worker_cpus": [
10
],
"guard_cpus": [
29
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 4,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
}
],
"receipts": {
"live-progress.json": {
"unix": 1795006694.7851024,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 5097.757695970009,
"cumulative_training_gpu_seconds": 53041.704901624005,
"postfit_started": false
},
"latest-cost-projection.json": {
"status": "PASS",
"rows": [
{
"id": 3,
"projected_gpu_seconds": 30960,
"basis": "continuing_projection_or_pending_cap"
},
{
"id": 4,
"projected_gpu_seconds": 13693.198552801914,
"basis": "continuing_projection_or_pending_cap"
}
],
"new_projected_gpu_seconds": 51853.198552801914,
"old_gpu_seconds_once": 47943.94719455298,
"cumulative_projected_gpu_seconds": 99797.1457473549,
"postfit_reserve": 7200
}
},
"workers": [
{
"job": "4",
"progress": {
"fold": 4,
"stage": "TRAIN",
"epoch": 46,
"optimizer_steps": 14087,
"wall_seconds": 2476.8090013740584,
"cpu_seconds": 2475.898949847,
"case": "00000986_0dd1386597104265ab67e6cf_step_009",
"epoch_parts": 137,
"epoch_nodes": 3867916,
"lr": 0.00091486437141827,
"loss": 0.00040039376472122967,
"gradient_norm_before_clip": 0.020746927708387375,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"completed_epochs": 45,
"recent10_epoch_seconds": [
56.72672600694932,
55.961798021919094,
54.08011715405155,
53.752001903019845,
52.82414131902624,
56.1399729748955,
55.80259197601117,
52.83759187697433,
55.623372485046275,
53.57573465502355
],
"median_recent10": 54.85174481954891,
"peak_allocated": 1461572608,
"peak_reserved": 1516240896
}
]
}
]
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,7 @@
{
"start": 1795006695.2355924,
"end": 1795006695.3000371,
"returncode": 0,
"stdout": "\n==============NVSMI LOG==============\n\nTimestamp : Wed Nov 18 20:58:15 2026\nDriver Version : 580.196.07\nCUDA Version : 13.0\n\nAttached GPUs : 4\nGPU 00000000:84:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 51 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp : 90 C\n GPU Slowdown Temp : 87 C\n GPU Max Operating Temp : 83 C\n GPU Target Temperature : N/A\n Memory Current Temp : 50 C\n Memory Max Operating Temp : 85 C\n\nGPU 00000000:85:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 51 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp : 90 C\n GPU Slowdown Temp : 87 C\n GPU Max Operating Temp : 83 C\n GPU Target Temperature : N/A\n Memory Current Temp : 50 C\n Memory Max Operating Temp : 85 C\n\nGPU 00000000:A4:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 55 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp :Line truncated
"stderr": ""
}
@@ -0,0 +1,123 @@
{
"unix": 1795006914.731002,
"benchmark_processes": [],
"benchmark_root_exists": false,
"existing_processes": [
{
"ppid": 2234,
"pid": 1476593,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
]
},
{
"ppid": 1476627,
"pid": 1476707,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"0",
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
]
},
{
"ppid": 1476628,
"pid": 1476726,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"1",
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
]
},
{
"ppid": 1598086,
"pid": 1598144,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"2",
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
]
},
{
"ppid": 2234,
"pid": 1650006,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
]
},
{
"ppid": 1650109,
"pid": 1650207,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/track_a_recovery_worker.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
"4"
]
}
],
"queues": {
"phase3-seed1-v2-2026-11-18": {
"pins": "PASS",
"live": {
"stage": "SEED1_TRAINING",
"unix": 1795006914.438547,
"active": [
{
"id": 0,
"guard_pid": 1476627
},
{
"id": 1,
"guard_pid": 1476628
},
{
"id": 2,
"guard_pid": 1598086
}
],
"terminal": [],
"pending": [
3,
4
],
"charged_training_gpu_seconds": 36725.94965413795,
"postfit_gpu_reserve": 7200
},
"complete": false,
"global_stop": false
},
"track-a-recovery-2026-11-18": {
"pins": "PASS",
"live": {
"unix": 1795006913.7070353,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 5535.601560306037,
"cumulative_training_gpu_seconds": 53479.548766289954,
"postfit_started": false
},
"complete": false,
"global_stop": false
}
},
"benchmark_ever_launched": false
}
@@ -0,0 +1,98 @@
"""Independent H2 throughput experiment: learning-only, no scientific scoring."""
import json
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked
from phase2_home_freeze_io import verify
BASE = Path('/home/frank14f/optane/MeltMind')
ROOT = BASE / 'artifacts/h2-concurrency-probe-2026-11-18'
GPU1 = 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'
def read(p):
return json.loads(Path(p).read_text())
def config(root):
root = Path(root)
if root.resolve() != ROOT:
raise PermissionError('Sole labelled probe root')
verify(root, 'implementation')
c = read(root / 'contract.json')
expected = dict(approval='1795006592.506519', gpu_uuid=GPU1,
concurrency=[1, 2, 3], epochs=5, warmup_epochs=1,
seed=0, gpu_seconds_cap=10800, no_retry=True,
training_fold=0, test_reads=False, gpu3_limit=1)
if any(c.get(k) != v for k, v in expected.items()):
raise PermissionError('Frozen experiment scope changed')
for p, h in {**c['code_sha256'], **c['source_sha256']}.items():
checked(p, h)
view = read(root / 'learning-view.json')
f = view['folds'][0]
if (len(view['folds']) != 1 or f['fold'] != 0 or f['validation'] or f['test']
or set(view['parts']) != set(f['learning'])
or set(view['geometry']) != set(f['learning'])):
raise PermissionError('Learning-only view required')
stats = read(c['standardization'])
if stats['parts'] != f['learning'] or stats['nodes'] != f['learning_nodes']:
raise ValueError('Normalization identity')
return c, view, stats
def charge(now, outside_seconds, jobs):
"""Sum each slot's wall seconds, plus coordinator time without slots and60s."""
values = [outside_seconds, now]
amounts = []
for j in jobs:
end = j.get('end', now)
values += [j['start'], end]
if end < j['start']:
raise ValueError('Nonmonotonic job clock')
amounts.append(end - j['start'])
if any(not math.isfinite(x) or x < 0 for x in values):
raise ValueError('Invalid accounting input')
return 60 + outside_seconds + sum(amounts)
def throughput(rows):
if not rows or any(r['completed_epochs'] != 5 for r in rows):
raise ValueError('Five complete epochs per copy required')
starts = [r['measure_start'] for r in rows]
ends = [r['measure_end'] for r in rows]
if any(not math.isfinite(x) for x in starts + ends) or any(b <= a for a, b in zip(starts, ends)):
raise ValueError('Invalid measurement interval')
return dict(total_epochs_per_hour=4 * len(rows) * 3600 / (max(ends) - min(starts)),
per_copy_epochs_per_hour=[4 * 3600 / (b - a) for a, b in zip(starts, ends)])
def choose(rows):
"""A larger N requires every preceding step's >=1.3 gain and margin pass."""
selected = None
for i, row in enumerate(rows, 1):
if row['concurrency'] != i:
raise ValueError('Ordered contiguous measurements required')
rate = row['total_epochs_per_hour']
if not math.isfinite(rate) or rate <= 0:
raise ValueError('Invalid throughput')
if not row['margin_pass'] or (i > 1 and rate < 1.3 * rows[i-2]['total_epochs_per_hour']):
break
selected = i
return selected
def resource_reasons(sample, c):
reasons = []
if sample['device_used'] > .8 * sample['device_total']:
reasons.append('DEVICE_20_PERCENT_MARGIN')
if sample['host_available'] < .2 * sample['host_total']:
reasons.append('HOST_RAM_20_PERCENT_MARGIN')
if sample['host_cpu_percent'] > 80:
reasons.append('HOST_CPU_20_PERCENT_MARGIN')
if sample['temperature'] >= c['temperature_limit_c']:
reasons.append('DEVICE_TEMPERATURE_LIMIT')
if any(v > c['worker_rss_bytes'] for v in sample['rss']):
reasons.append('WORKER_RSS_LIMIT')
if sample['foreign_gpu_pids']:
reasons.append('FOREIGN_GPU_PROCESS')
return reasons
@@ -0,0 +1,85 @@
"""Fresh H2/seed0, original200-epoch LR prefix, five learning-only epochs."""
import os
from pathlib import Path
import sys
import time
import traceback
import numpy as np
import psutil
import torch
from phase2_h1_worker_v2 import save
from phase2_h2_data import load_part, to_device
from phase2_h2_model import H2Geo, optimizer, lr_epoch
from phase2_h2_pilot_worker import seed, train_step, peaks
from h2_concurrency_probe_common import config
def run(root, n, index):
c, view, stats = config(root)
if n not in (1, 2, 3) or index not in range(n):
raise PermissionError('Unapproved copy')
phase = root / ('n' + str(n)); out = phase / str(index)
out.mkdir(exist_ok=False)
os.sched_setaffinity(0, [c['worker_cpus'][index]])
torch.set_num_threads(1); torch.set_num_interop_threads(1)
if os.environ.get('CUDA_VISIBLE_DEVICES') != c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF') != 'expandable_segments:True':
raise ValueError('GPU/allocator identity')
start = time.monotonic(); process = psutil.Process(); epoch = steps = 0
rows = []; measure_start = measure_end = None
def barrier(name):
while not (phase / name).exists():
if time.monotonic() - start >= c['job_wall_seconds'] - 30:
raise TimeoutError('Barrier exhausted own window')
time.sleep(.1)
try:
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.cuda.set_device(0); torch.cuda.init(); torch.cuda.reset_peak_memory_stats(); seed()
model = H2Geo().cuda(); opt = optimizer(model)
f = view['folds'][0]; parts = len(f['learning']); total = f['learning_nodes']
save(out / 'ready.json', dict(pid=os.getpid(), unix=time.time(), **peaks()))
barrier('go.json')
for epoch in range(1, 6):
if epoch == 2:
save(out / 'warm.json', dict(epoch=1, pid=os.getpid()))
barrier('measure-go.json'); measure_start = time.monotonic()
begin = time.monotonic(); seen = nodes = 0; load_s = transfer_s = step_s = 0.
io0 = process.io_counters(); cpu0 = time.process_time()
if any(abs(g['lr'] - lr_epoch(epoch)) > 1e-15 for g in opt.param_groups):
raise ValueError('Original learning-rate prefix changed')
order = np.random.default_rng(np.random.SeedSequence([0, epoch])).permutation(parts)
for k in order:
case = f['learning'][int(k)]
t = time.monotonic(); data = load_part(view, case, stats, 'learning', out)
load_s += time.monotonic() - t
t = time.monotonic(); data = to_device(data, 'cuda'); torch.cuda.synchronize()
transfer_s += time.monotonic() - t
t = time.monotonic(); result = train_step(model, opt, data, parts, total)
step_s += time.monotonic() - t
seen += 1; nodes += data['nodes']; steps += 1
save(out / 'progress.json', dict(epoch=epoch, parts=seen, optimizer_steps=steps,
rss=process.memory_info().rss, loss=result['loss'], **peaks()))
del data
if seen != parts or nodes != total:
raise ValueError('Incomplete learning epoch')
io1 = process.io_counters()
row = dict(epoch=epoch, seconds=time.monotonic()-begin,
load_prepare_seconds=load_s, transfer_seconds=transfer_s, train_step_seconds=step_s,
cpu_seconds=time.process_time()-cpu0,
io_delta={k: getattr(io1,k)-getattr(io0,k) for k in io0._fields}, **peaks())
rows.append(row); save(out / 'epochs.json', rows)
for group in opt.param_groups: group['lr'] = lr_epoch(epoch+1)
measure_end = time.monotonic()
save(out / 'result.json', dict(status='COMPLETE', completed_epochs=5, optimizer_steps=steps,
measure_start=measure_start, measure_end=measure_end, epochs=rows, worker_wall=time.monotonic()-start,
cpu_seconds=time.process_time(), test_reads=False, validation_reads=False,
checkpoint_saved=False, role='throughput_only_not_scientific_model', **peaks()))
except BaseException as exc:
save(out / 'failure.json', dict(status='FAILED', type=type(exc).__name__, reason=str(exc),
traceback=traceback.format_exc(), epoch=epoch, steps=steps,
**(peaks() if torch.cuda.is_initialized() else {})))
raise
if __name__ == '__main__':
run(Path(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]))
@@ -0,0 +1,26 @@
{
"unix": 1795006860.9747784,
"status": "WITHDRAWN_BEFORE_PREPARATION_TEST_OR_EXECUTION",
"reason": "User corrected intent: concurrency of distinct research ideas, not same-workload throughput optimization. Claude leads revised work assignments.",
"actual_work": [
"six read-only resource samples over25.5685s",
"two incomplete draft source files archived"
],
"gpu_workload_launched": false,
"learning_or_test_targets_read": false,
"tests_run": false,
"implementation_frozen": false,
"runtime_root_exists": false,
"files": [
{
"original": "/home/frank14f/optane/MeltMind/tools/h2_concurrency_probe_common.py",
"archive": "/home/frank14f/optane/MeltMind/artifacts/gpu-concurrency-assessment-2026-11-18/withdrawn-throughput-draft/h2_concurrency_probe_common.py",
"sha256": "a80e431e3bf4d246b0f17ab66ca6c8b4f78a3db82b632fd1d601acb7f69bad87"
},
{
"original": "/home/frank14f/optane/MeltMind/tools/h2_concurrency_probe_worker.py",
"archive": "/home/frank14f/optane/MeltMind/artifacts/gpu-concurrency-assessment-2026-11-18/withdrawn-throughput-draft/h2_concurrency_probe_worker.py",
"sha256": "1fb7aa10065977b55c71082adde6e15fc44e565353e11d3487dc29ec1a871072"
}
]
}