Bind one Greif training, custody and paired scoring entry for review
This commit is contained in:
1 parent
8810276beb
commit
3c82c074e8
9 files changed
+3419
No files matched your search
+1124
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 7,
|
||||
"actual_gpu_queries": 0,
|
||||
"real_target_arrays_read": 0,
|
||||
"scenarios": {
|
||||
"nominal": {
|
||||
"status": "PREDICTIONS_FROZEN_NO_TARGETS",
|
||||
"trigger": null,
|
||||
"stages": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
"first"
|
||||
],
|
||||
[
|
||||
1,
|
||||
0,
|
||||
"first"
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
"fit"
|
||||
],
|
||||
[
|
||||
0,
|
||||
1,
|
||||
"fit"
|
||||
],
|
||||
[
|
||||
1,
|
||||
0,
|
||||
"fit"
|
||||
],
|
||||
[
|
||||
1,
|
||||
1,
|
||||
"fit"
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
"predict"
|
||||
],
|
||||
[
|
||||
0,
|
||||
1,
|
||||
"predict"
|
||||
],
|
||||
[
|
||||
1,
|
||||
0,
|
||||
"predict"
|
||||
],
|
||||
[
|
||||
1,
|
||||
1,
|
||||
"predict"
|
||||
]
|
||||
],
|
||||
"cpu_child_spawns": 10
|
||||
},
|
||||
"busy": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "0/0/first IDLE_DEADLINE",
|
||||
"stages": [],
|
||||
"cpu_child_spawns": 0
|
||||
}
|
||||
},
|
||||
"contract_sha256": "f56e0d1a0e64fb46f2af6a731730308856083bdd18cfe030ab478b0cd841d539",
|
||||
"replay_log_sha256": "33f471940854a4ac2a988f0cfd12a857016c8aea97197d2d26ff6ca52e6c7684"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
test_actual_checkpoint_preserves_optimizer_for_second_update (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_first_cost_gate_counts_spent_and_blocks_over_budget (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_matching_drive_checksum_is_required (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_no_review_prevents_any_gpu_probe (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_replays_actual_production_guard_and_stage_order (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_scoring_decision_fixed_roster_and_named_hard_intersection (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
test_two_predictions_share_exactly_one_mock_target_open (test_track_b_gino_greif_combined.Combined) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 2.532s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,17 @@
|
||||
# GINO-Greif 训练、预测、保管与评分联合预审
|
||||
|
||||
此绑定采用已验收的适配器 `01d6581c`。固定 grouped folds 0/1、seeds 0/1、200 轮,以及原 H2 的 19 列、学习集标准化、FP64 `Vf`、折内冻结 `c` 和 PPS 采样。四个模型沿用 H2 的 AdamW、200 轮学习率日程、逐节点等权 MSE 和梯度裁剪。预测输出覆盖全部原节点,再按原始查询评分。 [Claude 裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795087203745169) · [联合合同](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/contract.json)
|
||||
|
||||
## 训练与成本
|
||||
|
||||
在同一次正式运行内先完成两折 seed 0 的首轮,保存模型、优化器、折内标准化和随机状态。两个实测均覆盖完整学习域才检查成本。通过后两条 seed 0 从第 2 轮续接,seed 1 从第 1 轮训练。首轮各记一次,未执行轮数每折为 399。投影为两折首轮已耗加各自余轮实测乘 1.25,再加未来训练设置 1,200 GPU 秒及四段预测各 1,800 GPU 秒。缺测则无续接,超本项 100,000 GPU 秒或 Phase 3 共 200 GPU 小时即停止。实际阶段账按统一时钟和实际守卫核验。GPU0–2 优先,GPU3 最后且独占。 [入口、成本和阶段门](../tools/track_b_gino_greif_entry.py)
|
||||
|
||||
## 保管与评分
|
||||
|
||||
同一入口的 `run` 完成训练和无目标预测,`pack` 将四份最终权重和 424 份预测归档。批准后,执行方用 Drive 连接器上传原包到用户的私有项目目录,再读取文件 ID、父目录、字节数和 SHA-256,核实项目内权限并登记。Git 只收小索引。只有完成终态、四模型与全预测哈希以及 Drive 回读均匹配时,`score` 才用 CPU 打开留出数据。每件实际目标只打开一次,供两种子共享;已有 H2 指标按冻结配对清单复用。 [训练与预测代码](../tools/track_b_gino_greif_worker.py) · [保管与评分代码](../tools/track_b_gino_greif_custody.py)
|
||||
|
||||
主裁决固定 212 件、78 家族。每件先在每个模型内等权平均两个种子的时场相对 L2,再在家族内平均,最后家族等权。对 GINO−H2 做 2,000 次配对家族 bootstrap,95% 区间上界低于零认定 GINO 更好,下界高于零认定 H2 更好,含零归为相当。末填、Kendall τ-b 和既有 34 难件名单在两折中的 12 件交集只作描述。主指标缺支持时保留缺项。 [预登记比较](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/comparison-contract.json)
|
||||
|
||||
## 当前验收
|
||||
|
||||
七项新增 CPU 回放通过。生产守卫用冻结合同顺序跑完两次首轮、四次训练续接和四次预测的合成子进程,忙卡零派发。另验第一轮优化器状态保留、缺测和超帽、Drive 校验、配对判决及单目标读取。真实 GPU 查询、训练、预测、上传和留出读取均为零。独立 PASS、ShapeNet 正式裁定、Yanqi 本项明确启动许可和足额实时 Phase 3 账均待采用。 [CPU 回执](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/cpu-replay.json)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""CPU replay for the reviewed Greif cycle, first-epoch continuity and one-read scorer."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from s2_gpu_idle_gate import wait_idle
|
||||
import track_b_gino_formal_guard as guard
|
||||
import track_b_gino_greif_custody as custody
|
||||
import track_b_gino_greif_entry as entry
|
||||
import track_b_gino_greif_worker as worker
|
||||
from track_b_gino_shapenet_entry import read, save, sha
|
||||
|
||||
|
||||
EVIDENCE={}
|
||||
|
||||
|
||||
class Combined(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.c=entry.contract()
|
||||
cls.contract_path=entry.ROOT/'contract.json'
|
||||
cls.comparison=read(entry.ROOT/'comparison-contract.json')
|
||||
|
||||
def test_replays_actual_production_guard_and_stage_order(self):
|
||||
for busy in (False,True):
|
||||
with self.subTest(busy=busy), TemporaryDirectory() as directory:
|
||||
root=Path(directory)
|
||||
(root/'contract.json').write_bytes(self.contract_path.read_bytes())
|
||||
save(root/'stage-authorization.json',dict(test=True))
|
||||
path=root/'track_b_gino_greif_worker.py'
|
||||
path.write_text(
|
||||
'import json,sys,pathlib\n'
|
||||
'root=pathlib.Path(__file__).parent\n'
|
||||
'fold,seed,phase=sys.argv[1:]\n'
|
||||
'stage=root/f"fold-{fold}"/f"seed-{seed}"/phase\n'
|
||||
'p=stage/"worker-records"; p.mkdir()\n'
|
||||
'spec=json.loads((root/"contract.json").read_text())["folds"][fold]\n'
|
||||
'obj=dict(learning_cases=spec["counts"]["learning"],learning_nodes=spec["learning_nodes"],'
|
||||
'first_epoch_seconds=.01,files=[],peak_allocated_bytes=1,peak_reserved_bytes=2)\n'
|
||||
'(p/"temp").write_text(json.dumps(obj)); (p/"temp").rename(p/"result.json")\n')
|
||||
observed=[]
|
||||
elapsed=[0.]
|
||||
card=entry.DEVICES[0]
|
||||
|
||||
def idle(uuid):
|
||||
return dict(uuid=uuid,idle=True,occupants=[],memory_MiB=0,unix=10.)
|
||||
|
||||
def device(uuid):
|
||||
return dict(idle(uuid),idle=not busy)
|
||||
|
||||
def simulated_idle(uuid, limit, record, *, sample):
|
||||
return wait_idle(uuid,limit,record,sample=sample,clock=lambda:elapsed[0],
|
||||
sleep=lambda s:elapsed.__setitem__(0,elapsed[0]+s))
|
||||
|
||||
def validate(_,fold,seed,phase):
|
||||
observed.append((fold,seed,phase))
|
||||
return sha(entry.path(fold,seed,phase)/'worker-records/result.json')
|
||||
|
||||
with (patch.object(entry,'ROOT',root),patch.object(entry,'SOURCE',root),
|
||||
patch.object(entry,'snapshot',side_effect=idle),
|
||||
patch.object(entry,'validate_result',side_effect=validate),
|
||||
patch.object(guard,'snapshot',side_effect=device),
|
||||
patch.object(guard,'wait_idle',side_effect=simulated_idle),
|
||||
patch.object(guard,'dispatch_environment',
|
||||
side_effect=lambda u,env:(dict(env,CUDA_VISIBLE_DEVICES='0'),[]))):
|
||||
result=entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
|
||||
with self.assertRaises(FileExistsError):
|
||||
entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
|
||||
terminal=read(root/'complete.json')
|
||||
p=root/'fold-0/seed-0/first'
|
||||
if busy:
|
||||
self.assertEqual(result,1)
|
||||
self.assertIn('IDLE_DEADLINE',terminal['trigger'])
|
||||
self.assertFalse((p/'actual-worker-spawn.json').is_file())
|
||||
self.assertFalse((root/'predictions-frozen.json').exists())
|
||||
self.assertEqual(observed,[])
|
||||
else:
|
||||
expected=[(0,0,'first'),(1,0,'first')]+[(f,s,p) for p in ('fit','predict') for f,s in entry.JOBS]
|
||||
self.assertEqual(result,0)
|
||||
self.assertEqual(observed,expected)
|
||||
self.assertTrue(read(root/'first-epoch-cost-gate.json')['release_epoch2'])
|
||||
for f,s,phase in observed:
|
||||
event=read(root/f'fold-{f}'/f'seed-{s}'/phase/'complete.json')
|
||||
self.assertEqual(event['physical_uuid'],card)
|
||||
self.assertEqual([v['event'] for v in event['lifecycle']],['leader_reaped'])
|
||||
self.assertFalse((root/'score').exists())
|
||||
EVIDENCE['busy' if busy else 'nominal']=dict(
|
||||
status=terminal['status'],trigger=terminal['trigger'],stages=observed,
|
||||
cpu_child_spawns=len(list(root.rglob('actual-worker-spawn.json'))))
|
||||
|
||||
def test_first_cost_gate_counts_spent_and_blocks_over_budget(self):
|
||||
common=dict(seed=0,epoch=1,heldout_target_reads=0,booked_gpu_seconds=400.)
|
||||
p={k:dict(common,fold=int(k),epoch_seconds=70.,learning_nodes=r['learning_nodes'],
|
||||
learning_cases=r['counts']['learning']) for k,r in self.c['folds'].items()}
|
||||
gate=entry.cost_gate(self.c,p,495434.9566447616,820.)
|
||||
self.assertTrue(gate['release_epoch2'])
|
||||
self.assertEqual(gate['charged_first_epochs_and_setup_gpu_seconds'],800.)
|
||||
self.assertEqual(gate['remaining_train_gpu_seconds'],(70+70)*399)
|
||||
self.assertFalse(entry.cost_gate(self.c,{'0':p['0']},495434.9566447616,820.)['release_epoch2'])
|
||||
p['1']['epoch_seconds']=185.
|
||||
self.assertEqual(entry.cost_gate(self.c,p,495434.9566447616,820.)['status'],'STOP_PROJECTED_CAP')
|
||||
p['1']['epoch_seconds']=70.
|
||||
self.assertFalse(entry.cost_gate(self.c,p,680000.,820.)['release_epoch2'])
|
||||
|
||||
def test_actual_checkpoint_preserves_optimizer_for_second_update(self):
|
||||
torch.manual_seed(7)
|
||||
model=torch.nn.Linear(2,1)
|
||||
opt=torch.optim.AdamW(model.parameters())
|
||||
x=torch.tensor([[3.,7.]])
|
||||
def step(m,o):
|
||||
o.zero_grad()
|
||||
m(x).square().sum().backward()
|
||||
o.step()
|
||||
|
||||
step(model,opt)
|
||||
with TemporaryDirectory() as directory, patch.object(torch.cuda,'get_rng_state',return_value=torch.get_rng_state()):
|
||||
path=Path(directory)/'epoch-1.pt'
|
||||
worker.checkpoint(path,model,opt,1,0,0,{'only_learning':True})
|
||||
saved=worker.restore(dict(path=str(path),sha256=sha(path)),0,0,1)
|
||||
copy=torch.nn.Linear(2,1)
|
||||
copy.load_state_dict(saved['model'])
|
||||
resumed=torch.optim.AdamW(copy.parameters())
|
||||
resumed.load_state_dict(saved['optimizer'])
|
||||
step(model,opt)
|
||||
step(copy,resumed)
|
||||
self.assertTrue(all(torch.equal(a,b) for a,b in zip(model.parameters(),copy.parameters())))
|
||||
|
||||
def test_scoring_decision_fixed_roster_and_named_hard_intersection(self):
|
||||
def metrics(x):
|
||||
return {k:dict(value=x,reason=None) for k in custody.METRICS}
|
||||
|
||||
for g,h,decision in ((.06,.07,'GINO_BETTER'),(.07,.07,'COMPARABLE'),(.08,.07,'H2_BETTER')):
|
||||
rows=[dict(r,gino=[metrics(g+.001),metrics(g-.001)],h2=[metrics(h),metrics(h)])
|
||||
for r in self.comparison['roster']]
|
||||
result=custody.summary(rows,self.comparison)
|
||||
self.assertEqual(result['field']['decision'],decision)
|
||||
self.assertEqual(result['field']['components'],78)
|
||||
self.assertEqual(result['hard_subset']['frozen_total'],34)
|
||||
rows[0]['gino'][0]['relative_l2']['value']=None
|
||||
self.assertEqual(custody.summary(rows,self.comparison)['field']['decision'],'INCOMPLETE_PAIRED_SUPPORT')
|
||||
with self.assertRaises(ValueError):
|
||||
custody.summary(rows[:-1],self.comparison)
|
||||
|
||||
def test_matching_drive_checksum_is_required(self):
|
||||
with TemporaryDirectory() as directory, patch.object(entry,'ROOT',Path(directory)):
|
||||
root=Path(directory)
|
||||
for n in ('contract.json','models-frozen.json','predictions-frozen.json'):
|
||||
save(root/n,[])
|
||||
path=root/'gino-greif-models-and-predictions.zip'
|
||||
path.write_bytes(b'fixture')
|
||||
bundle=dict(path=str(path),sha256=sha(path),bytes=path.stat().st_size,contract_sha256=sha(root/'contract.json'),
|
||||
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json'))
|
||||
save(root/'drive-upload-index.json',bundle)
|
||||
auth=dict(private_drive_folder_id='folder-fixture')
|
||||
receipt=dict(sha256Checksum='wrong',size=str(bundle['bytes']),parent_id='folder-fixture',
|
||||
file_id='fixture',project_private=True,contract_sha256=bundle['contract_sha256'])
|
||||
save(root/'drive-readback.json',receipt)
|
||||
with self.assertRaises(PermissionError):
|
||||
custody.verify_drive(auth)
|
||||
save(root/'drive-readback.json',dict(receipt,sha256Checksum=sha(path)))
|
||||
self.assertEqual(custody.verify_drive(auth)['file_id'],'fixture')
|
||||
|
||||
def test_two_predictions_share_exactly_one_mock_target_open(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
root=Path(directory)
|
||||
raw=np.array([20,24,28])
|
||||
coords=np.array([[0.,0.,0.],[1.,0.,0.],[2.,0.,0.]])
|
||||
q=dict(raw_ids=raw,counts=np.array([1,1,1]),near_wall=np.array([True,False,False]),coordinates_over_Rg=coords)
|
||||
row=dict(fold=0,case='0003_fixture',family='test-family')
|
||||
forecasts={}
|
||||
for seed in (0,1):
|
||||
p=root/(str(seed)+'.npz')
|
||||
np.savez(p,seconds=np.array([1.,2.,4.]),raw_ids=raw,query_raw_ids=raw,
|
||||
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=coords)
|
||||
forecasts[0,seed,row['case']]=dict(family=row['family'],path=str(p),sha256=sha(p))
|
||||
pilot=dict(parts={row['case']:dict(output_sha256={'seconds.npy':'fixture'})})
|
||||
ref={row['case']:dict(fold=0,component_id=row['family'],seed0={},seed1={})}
|
||||
with ((root/'log').open('x') as log, patch.object(custody,'prediction_inputs',return_value=q),
|
||||
patch.object(custody,'cached',return_value=raw),
|
||||
patch.object(custody,'target_array',return_value=np.array([1.,2.,3.])) as target):
|
||||
result=custody.one_case(pilot,{},row,forecasts,ref,log)
|
||||
target.assert_called_once_with(pilot,row['case'])
|
||||
self.assertEqual(len(result['gino']),2)
|
||||
self.assertEqual(len((root/'log').read_text().splitlines()),1)
|
||||
|
||||
def test_no_review_prevents_any_gpu_probe(self):
|
||||
with patch.object(entry,'snapshot') as probe:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
entry.authorize(self.c)
|
||||
probe.assert_not_called()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Drive-backed Greif safekeeping and one guarded custody read per test case."""
|
||||
|
||||
from collections import defaultdict
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import time
|
||||
from zipfile import ZIP_STORED, ZipFile
|
||||
|
||||
import numpy as np
|
||||
|
||||
from phase2_h2_data import cached
|
||||
from s2_custody_common import prediction_inputs
|
||||
from s2_custody_statistics import part_metrics
|
||||
import track_b_gino_greif_entry as entry
|
||||
from track_b_gino_greif_prepare import PILOT, QUERY, PINS
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
METRICS=('relative_l2','last_location','kendall_tau_b')
|
||||
|
||||
|
||||
def frozen(c):
|
||||
root=entry.ROOT
|
||||
terminal=read(root/'complete.json')
|
||||
if (terminal['status'],terminal['contract_sha256'],terminal['test_targets_read']) != (
|
||||
'PREDICTIONS_FROZEN_NO_TARGETS',sha(root/'contract.json'),False) or not 0<terminal['charged_gpu_seconds']<=c['budgets']['gpu_cap']:
|
||||
raise PermissionError('A capped completed prediction terminal must precede custody')
|
||||
models=[]
|
||||
predictions=[]
|
||||
for fold,seed in entry.JOBS:
|
||||
models.extend(entry.stage_terminal(fold,seed,'fit')['files'])
|
||||
predictions.extend(entry.stage_terminal(fold,seed,'predict')['files'])
|
||||
if models!=read(root/'models-frozen.json') or predictions!=read(root/'predictions-frozen.json'):
|
||||
raise ValueError('Final sealed indexes differ from accepted terminals')
|
||||
if len(models)!=4 or len(predictions)!=424 or len({r['path'] for r in models+predictions})!=428:
|
||||
raise ValueError('Four final weights and 424 forecasts are required')
|
||||
for item in models+predictions:
|
||||
checked(item['path'],item['sha256'])
|
||||
if not Path(item['path']).resolve().is_relative_to(root):
|
||||
raise ValueError('Artifact path leaves the frozen run root')
|
||||
return models,predictions
|
||||
|
||||
|
||||
def pack(c):
|
||||
root=entry.ROOT
|
||||
models,predictions=frozen(c)
|
||||
dest=root/'gino-greif-models-and-predictions.zip'
|
||||
with ZipFile(dest,'x',compression=ZIP_STORED) as archive:
|
||||
for item in models+predictions:
|
||||
archive.write(item['path'],Path(item['path']).relative_to(root))
|
||||
for name in ('models-frozen.json','predictions-frozen.json','contract.json'):
|
||||
archive.write(root/name,name)
|
||||
with ZipFile(dest,'r') as archive:
|
||||
for item in models+predictions:
|
||||
key=str(Path(item['path']).relative_to(root))
|
||||
with archive.open(key) as stream:
|
||||
actual=hashlib.sha256()
|
||||
for data in iter(lambda:stream.read(1024**2),b''):
|
||||
actual.update(data)
|
||||
if actual.hexdigest()!=item['sha256']:
|
||||
raise ValueError('Archived bytes differ from the frozen file index')
|
||||
save(root/'drive-upload-index.json',dict(status='READY_FOR_PRIVATE_PROJECT_DRIVE',path=str(dest),
|
||||
bytes=dest.stat().st_size,sha256=sha(dest),contract_sha256=sha(root/'contract.json'),
|
||||
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json')))
|
||||
return 0
|
||||
|
||||
|
||||
def verify_drive(auth):
|
||||
root=entry.ROOT
|
||||
bundle=read(root/'drive-upload-index.json')
|
||||
path=checked(bundle['path'],bundle['sha256'])
|
||||
if (path!=root/'gino-greif-models-and-predictions.zip'
|
||||
or path.stat().st_size!=bundle['bytes']
|
||||
or bundle['contract_sha256']!=sha(root/'contract.json')
|
||||
or bundle['model_index_sha256']!=sha(root/'models-frozen.json')
|
||||
or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')):
|
||||
raise ValueError('Local archive no longer matches the fixed models and forecasts')
|
||||
receipt=read(root/'drive-readback.json')
|
||||
# Drive file metadata, recorded by the authorized connector from the private project folder.
|
||||
if (receipt.get('sha256Checksum'),receipt.get('size'),receipt.get('parent_id')) != (
|
||||
bundle['sha256'],str(bundle['bytes']),auth['private_drive_folder_id']):
|
||||
raise PermissionError('Private Drive readback SHA, size and folder are required before scoring')
|
||||
if (not receipt.get('file_id') or receipt.get('project_private') is not True
|
||||
or receipt.get('contract_sha256')!=bundle['contract_sha256']):
|
||||
raise PermissionError('Private Drive identity required before opening targets')
|
||||
return receipt
|
||||
|
||||
|
||||
def bootstrap(rows, metric, fixed_support=None):
|
||||
values=defaultdict(list)
|
||||
unavailable=[]
|
||||
for row in rows:
|
||||
scores=[row[arm][seed][metric]['value'] for arm in ('gino','h2') for seed in (0,1)]
|
||||
if any(v is None or not np.isfinite(v) for v in scores):
|
||||
unavailable.append(row['case'])
|
||||
continue
|
||||
values[row['family']].append(((scores[0]+scores[1])/2,(scores[2]+scores[3])/2))
|
||||
absent=set(unavailable)
|
||||
rejected={r['family'] for r in rows if r['case'] in absent}
|
||||
family=np.array([np.mean(values[k],axis=0) for k in sorted(set(values)-rejected)])
|
||||
support=dict(parts=len(rows),components=len(family),undefined_cases=unavailable)
|
||||
if not len(family) or (fixed_support is not None and (len(rows),len(family))!=fixed_support):
|
||||
return dict(**support,decision='INCOMPLETE_PAIRED_SUPPORT',ci95=None)
|
||||
delta=family[:,0]-family[:,1]
|
||||
rng=np.random.default_rng(2026111408)
|
||||
draw=rng.integers(len(family),size=(2000,len(family)))
|
||||
low,high=map(float,np.percentile(delta[draw].mean(axis=1),[2.5,97.5]))
|
||||
return dict(**support,gino=float(family[:,0].mean()),h2=float(family[:,1].mean()),delta=float(delta.mean()),
|
||||
ci95=[low,high],replicates=2000,seed=2026111408)
|
||||
|
||||
|
||||
def summary(rows, compare):
|
||||
if sorted((r['fold'],r['case'],r['family']) for r in rows)!=sorted(
|
||||
(r['fold'],r['case'],r['family']) for r in compare['roster']):
|
||||
raise ValueError('Exact two-fold paired roster required')
|
||||
primary=bootstrap(rows,'relative_l2',fixed_support=(212,78))
|
||||
if primary['ci95'] is not None:
|
||||
low,high=primary['ci95']
|
||||
primary['decision']='GINO_BETTER' if high<0 else 'H2_BETTER' if low>0 else 'COMPARABLE'
|
||||
hard=[r for r in rows if r['case'] in compare['hard34']]
|
||||
return dict(field=primary,secondary={metric:bootstrap(rows,metric) for metric in METRICS[1:]},
|
||||
hard_subset=dict(frozen_total=34,two_fold_cases=[r['case'] for r in hard],
|
||||
descriptive={metric:bootstrap(hard,metric) for metric in METRICS}))
|
||||
|
||||
|
||||
def target_array(pilot,case):
|
||||
item=pilot['parts'][case]
|
||||
file=BASE/item['output']/'seconds.npy'
|
||||
blob=file.read_bytes()
|
||||
if hashlib.sha256(blob).hexdigest()!=item['output_sha256']['seconds.npy']:
|
||||
raise ValueError('Pinned held-out bytes differ')
|
||||
return np.load(io.BytesIO(blob),allow_pickle=False)
|
||||
|
||||
|
||||
def one_case(pilot, meta, row, forecasts, reference, log):
|
||||
case=row['case']
|
||||
fold=row['fold']
|
||||
q=prediction_inputs(meta)
|
||||
raw=cached(pilot,case,'raw_ids')
|
||||
ix=np.searchsorted(raw,q['raw_ids'])
|
||||
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
|
||||
raise ValueError('Frozen PPS coordinates differ')
|
||||
new=[]
|
||||
for seed in (0,1):
|
||||
f=forecasts[fold,seed,case]
|
||||
if f['family']!=row['family']:
|
||||
raise ValueError('Predicted family differs')
|
||||
with np.load(checked(f['path'],f['sha256']),allow_pickle=False) as z:
|
||||
seconds=z['seconds']
|
||||
if (seconds.dtype!=np.float64 or seconds.shape!=raw.shape or not np.isfinite(seconds).all()
|
||||
or not np.array_equal(raw,z['raw_ids'])
|
||||
or any(not np.array_equal(q[name],z['query_raw_ids' if name=='raw_ids' else name])
|
||||
for name in ('counts','near_wall','coordinates_over_Rg','raw_ids'))):
|
||||
raise ValueError('Frozen full node or PPS support differs')
|
||||
new.append(seconds[ix])
|
||||
old=reference[case]
|
||||
if (old['fold'],old['component_id'])!=(fold,row['family']):
|
||||
raise ValueError('H2 score pairing differs')
|
||||
log.write(json.dumps(dict(case=case,fold=fold,role='test',sha256=pilot['parts'][case]['output_sha256']['seconds.npy']))+'\n')
|
||||
log.flush()
|
||||
truth=target_array(pilot,case)
|
||||
if truth.shape!=raw.shape:
|
||||
raise ValueError('Original held-out node support differs')
|
||||
gino=[part_metrics(p,truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],
|
||||
int(case.split('_')[0])) for p in new]
|
||||
return dict(row,gino=gino,h2=[old['seed0'],old['seed1']])
|
||||
|
||||
|
||||
def score(c):
|
||||
auth,_=entry.authorize(c)
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!='':
|
||||
raise PermissionError('Custody scoring must run with CUDA hidden')
|
||||
ceiling=resource.getrlimit(resource.RLIMIT_CPU)[1]
|
||||
limit=c['budgets']['score_cpu_seconds']
|
||||
if ceiling!=resource.RLIM_INFINITY:
|
||||
limit=min(limit,ceiling)
|
||||
resource.setrlimit(resource.RLIMIT_CPU,(limit,limit))
|
||||
_,predictions=frozen(c)
|
||||
drive=verify_drive(auth)
|
||||
compare=read(checked(c['comparison_contract'],c['comparison_contract_sha256']))
|
||||
rosters={(r['fold'],r['case']) for r in compare['roster']}
|
||||
new={(r['fold'],r['seed'],r['case']):r for r in predictions}
|
||||
if set(new)!={(fold,seed,case) for fold,case in rosters for seed in (0,1)}:
|
||||
raise ValueError('Frozen four-set test support differs')
|
||||
prior=read(checked(compare['h2_score']['path'],compare['h2_score']['sha256']))
|
||||
reference={r['case']:r for r in prior['rows']}
|
||||
if prior['status']!='COMPLETE' or len(reference)!=552 or not {r['case'] for r in compare['roster']}<=set(reference):
|
||||
raise ValueError('H2 frozen score support differs')
|
||||
pilot=read(checked(PILOT,PINS[str(PILOT)]))
|
||||
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
|
||||
out=entry.ROOT/'score'
|
||||
out.mkdir(exist_ok=False)
|
||||
save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'],
|
||||
manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),before_test_target_reads=True))
|
||||
results=[]
|
||||
with (out/'heldout-target-access.jsonl').open('x') as log:
|
||||
for row in compare['roster']:
|
||||
results.append(one_case(pilot,query[row['case']],row,new,reference,log))
|
||||
save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results,
|
||||
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id']))
|
||||
return 0
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Reviewed GINO-Greif training, predictions and Drive-backed custody entry."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from s2_gpu_idle_gate import snapshot
|
||||
import track_b_gino_greif_prepare as prep
|
||||
from track_b_gino_formal_guard import guard
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = prep.ROOT / 'combined-review'
|
||||
SOURCE = Path(__file__).resolve().parent
|
||||
ADAPTER = prep.REVIEW_ROOT / 'preparation-binding.json'
|
||||
ADAPTER_SHA = '01d6581c33a6c26d1965acb13248bdb741de892e5929fff08a7e4af431f00d0b'
|
||||
H2_SCORE = prep.H2_REFERENCE[1] / 'score/result.json'
|
||||
H2_SCORE_SHA = 'd259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4'
|
||||
HARD = BASE / 'artifacts/phase3-h2-shared-miss-attributes-2026-11-19/by-part.jsonl'
|
||||
DEVICES = ('GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a')
|
||||
JOBS = ((0,0),(0,1),(1,0),(1,1))
|
||||
CAP = 100000
|
||||
PRED_SECONDS = 1800
|
||||
FIT_SETUP_RESERVE = 1200
|
||||
FIRST_STAGE_MAX = 3600
|
||||
|
||||
|
||||
def comparison(candidate):
|
||||
rows = [json.loads(line) for line in HARD.read_text().splitlines()]
|
||||
hard = [r['case'] for r in rows if r['shared_miss']]
|
||||
if len(hard) != 34 or len(set(hard)) != 34:
|
||||
raise ValueError('The original named hard cohort is required')
|
||||
query = read(checked(prep.QUERY, prep.PINS[str(prep.QUERY)]))
|
||||
family = {r['case']:r['component_id'] for r in query['rows'] if r['included']}
|
||||
roster = [dict(fold=int(fold), case=case, family=family[case])
|
||||
for fold,data in candidate['folds'].items() for case in data['parts']['test']]
|
||||
if len({r['case'] for r in roster}) != 212 or len({r['family'] for r in roster}) != 78:
|
||||
raise ValueError('The fixed 212-part / 78-family support is required')
|
||||
return dict(roster=roster, hard34=hard, frozen_hard_source=dict(path=str(HARD),sha256=sha(HARD)),
|
||||
h2_score=dict(path=str(H2_SCORE),sha256=H2_SCORE_SHA), method='Paired component bootstrap of GINO minus H2',
|
||||
metric='field relative L2; mean of seed0 and seed1 per arm, then case and component means',
|
||||
confidence=.95, replicates=2000, bootstrap_seed=2026111408,
|
||||
gates={'high_below_zero':'GINO_BETTER','contains_zero':'COMPARABLE','low_above_zero':'H2_BETTER'},
|
||||
secondary=['last_location/Rg','Kendall tau-b','intersection of named hard34 and two-fold test'],
|
||||
custody='each of the 212 original targets once for both new predictions, after all four checkpoints and predictions frozen')
|
||||
|
||||
|
||||
def expected_contract():
|
||||
c = read(checked(ADAPTER, ADAPTER_SHA))
|
||||
for path,h in c['code_sha256'].items():
|
||||
checked(path,h)
|
||||
base = prep.reference.contract()
|
||||
comparison_path = ROOT / 'comparison-contract.json'
|
||||
if read(comparison_path) != comparison(c):
|
||||
raise ValueError('Pre-registered Greif comparison differs')
|
||||
code=dict(base['code_sha256'],**c['code_sha256'])
|
||||
for name in ('track_b_gino_greif_entry.py','track_b_gino_greif_worker.py','track_b_gino_greif_custody.py',
|
||||
'phase2_h2_model.py','phase2_h2_data.py','s2_custody_common.py','s2_custody_statistics.py'):
|
||||
code[str(SOURCE/name)] = sha(SOURCE/name)
|
||||
return dict(handoff='h-08c2a2a5',mode='gino-greif-combined',adapter_sha256=ADAPTER_SHA,
|
||||
comparison_contract=str(comparison_path),comparison_contract_sha256=sha(comparison_path),code_sha256=code,
|
||||
folds=c['folds'],jobs=[list(x) for x in JOBS],epochs=c['epochs'],environment=base['environment'],
|
||||
fixed_gino_reference=str(prep.reference.ROOT),phase3_anchor=c['projection'],limits=base['limits'],
|
||||
device_order=list(DEVICES),gpu3_single_task=True,
|
||||
budgets=dict(gpu_cap=CAP,phase3_cap=prep.CAP,max_first_stage=FIRST_STAGE_MAX,
|
||||
per_prediction=PRED_SECONDS,train_setup_reserve=FIT_SETUP_RESERVE,score_cpu_seconds=3600),
|
||||
run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits',
|
||||
'freeze models','form four target-free prediction sets','freeze predictions',
|
||||
'verified private Drive transfer','one controlled custody pass'],
|
||||
drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only'},
|
||||
no_retry=True)
|
||||
|
||||
|
||||
def contract():
|
||||
c = read(ROOT/'contract.json')
|
||||
if c != expected_contract():
|
||||
raise PermissionError('Exact reviewed Greif contract differs')
|
||||
return c
|
||||
|
||||
|
||||
def authorize(c):
|
||||
validate_review(ROOT)
|
||||
a=read(ROOT/'stage-authorization.json')
|
||||
if (a.get('decision'),a.get('mode'),a.get('contract_sha256'),a.get('review_sha256')) != (
|
||||
'APPROVED',c['mode'],sha(ROOT/'contract.json'),sha(ROOT/'review.json')):
|
||||
raise PermissionError('Exact review plus separate stage authorization required')
|
||||
if (not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id')
|
||||
or not a.get('private_drive_folder_id')):
|
||||
raise PermissionError('Direct user launch and ShapeNet verdict are required')
|
||||
verdict=read(checked(a['shape_verdict']['path'],a['shape_verdict']['sha256']))
|
||||
if (verdict['contract_sha256']!=prep.reference.sha(prep.reference.ROOT/'contract.json')
|
||||
or verdict['decision'] not in ('REPRODUCED','NOT_REPRODUCED')
|
||||
or verdict['claude_ruling_id']!=a['claude_shape_verdict_id']):
|
||||
raise PermissionError('Exact final ShapeNet verdict receipt required')
|
||||
latest=read(checked(a['phase3_ledger']['path'],a['phase3_ledger']['sha256']))
|
||||
stamp=read(ROOT/'started.json')['unix'] if (ROOT/'started.json').exists() else time.time()
|
||||
# Charge each currently active program at its full un-settled reserve.
|
||||
keys={'earlier_phase3_settled','gino_old_measurement','gino_formal','h2_pressure','other'}
|
||||
figures=latest['gpu_seconds']
|
||||
p=c['phase3_anchor']
|
||||
if (latest['phase3_cap_gpu_seconds']!=prep.CAP or not 0<=stamp-latest['checked_unix']<=900
|
||||
or set(figures)!=keys
|
||||
or figures['earlier_phase3_settled']<p['settled_phase3_through_1b_gpu_seconds']
|
||||
or figures['gino_old_measurement']!=p['gino_measurement_once_gpu_seconds']
|
||||
or figures['gino_formal']<p['gino_shapenet_formal_reserve_gpu_seconds']
|
||||
or figures['h2_pressure']<p['h2_pressure_unsettled_max_gpu_seconds']
|
||||
or any(type(x) not in (int,float) or not math.isfinite(x) or x<0 for x in figures.values())
|
||||
or sum(figures.values())+CAP>prep.CAP
|
||||
or type(a['dispatch_before_unix']) not in (float,int) or not math.isfinite(a['dispatch_before_unix'])):
|
||||
raise PermissionError('Live phase-3 ledger cannot book the declared Greif cap')
|
||||
return a, sum(figures.values())
|
||||
|
||||
|
||||
def path(fold, seed, phase):
|
||||
return ROOT/f'fold-{fold}'/f'seed-{seed}'/phase
|
||||
|
||||
|
||||
def stage_terminal(fold, seed, phase):
|
||||
root = path(fold,seed,phase)
|
||||
s = read(root/'complete.json')
|
||||
if (s['status'],s['fold'],s['seed'],s['phase'],s['contract_sha256']) != (
|
||||
'COMPLETE',fold,seed,phase,sha(ROOT/'contract.json')):
|
||||
raise ValueError('Stage terminal differs')
|
||||
return read(checked(root/'worker-records/result.json',s['result_sha256']))
|
||||
|
||||
|
||||
def files_for(result):
|
||||
entries=result['files']
|
||||
for row in entries:
|
||||
checked(row['path'],row['sha256'])
|
||||
return entries
|
||||
|
||||
|
||||
def validate_result(c, fold, seed, phase):
|
||||
root=path(fold,seed,phase)
|
||||
result=root/'worker-records/result.json'
|
||||
obj=read(result)
|
||||
ticket=read(root/'started.json')
|
||||
spawn=read(root/'actual-worker-spawn.json')
|
||||
uuid=DEVICES[int(ticket['ordinal'])]
|
||||
if (spawn['physical_uuid']!=uuid or spawn['ordinal']!=ticket['ordinal']
|
||||
or read(root/'worker-device-verified.json') != dict(
|
||||
process=spawn['process'],ordinal=ticket['ordinal'],physical_uuid=uuid,observed_uuids=[uuid])
|
||||
or (obj['status'],obj['fold'],obj['seed'],obj['phase'],obj['test_targets_read']) !=
|
||||
('COMPLETE',fold,seed,phase,False)):
|
||||
raise ValueError('Worker, physical card or target scope differs')
|
||||
files_for(obj)
|
||||
model=c['folds'][str(fold)]
|
||||
if phase=='predict':
|
||||
if [r['case'] for r in obj['files']] != model['parts']['test']:
|
||||
raise ValueError('Test prediction roster differs')
|
||||
else:
|
||||
if (obj['epochs'],obj['learning_cases'],obj['learning_nodes']) != (
|
||||
1 if phase=='first' else c['epochs'],model['counts']['learning'],model['learning_nodes']):
|
||||
raise ValueError('Full fold epoch coverage required')
|
||||
return sha(result)
|
||||
|
||||
|
||||
def run_stage(c, auth, fold, seed, phase, deadline):
|
||||
if time.time()>=auth['dispatch_before_unix']:
|
||||
raise RuntimeError('APPROVED_DISPATCH_WINDOW_EXPIRED')
|
||||
ordinal=next((i for i,u in enumerate(DEVICES) if snapshot(u)['idle']),None)
|
||||
if ordinal is None:
|
||||
raise RuntimeError('NO_IDLE_APPROVED_CARD')
|
||||
started=time.monotonic()
|
||||
if deadline-started<=c['limits']['cleanup_seconds']:
|
||||
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
||||
root=path(fold,seed,phase)
|
||||
root.mkdir(parents=True,exist_ok=False)
|
||||
ticket=dict(fold=fold,seed=seed,phase=phase,ordinal=str(ordinal),contract_sha256=sha(ROOT/'contract.json'),
|
||||
booked_gpu_seconds=deadline-started,guard_started_monotonic=started,
|
||||
dispatch_before_unix=auth['dispatch_before_unix'])
|
||||
save(root/'started.json',ticket)
|
||||
command=[c['environment']['python'],'-B',str(SOURCE/'track_b_gino_greif_worker.py'),str(fold),str(seed),phase]
|
||||
receipt=guard(root,command,uuid=DEVICES[ordinal],limits=dict(c['limits'],cpu_seconds=8*(deadline-started)),
|
||||
deadline=deadline,ticket=ticket,validate_result=lambda:validate_result(c,fold,seed,phase))
|
||||
if receipt['status']!='COMPLETE':
|
||||
raise RuntimeError(f"{fold}/{seed}/{phase} {receipt['trigger']}")
|
||||
return stage_terminal(fold,seed,phase), receipt['charged_gpu_seconds']
|
||||
|
||||
|
||||
def cost_gate(c, receipts, phase_booked, elapsed):
|
||||
candidate=dict(folds=c['folds'],epochs=c['epochs'],projection=dict(
|
||||
accounted_gpu_seconds=phase_booked,upper_remaining_before_other_bookings_gpu_seconds=prep.CAP-phase_booked))
|
||||
stages=sum(r['booked_gpu_seconds'] for r in receipts.values())
|
||||
overhead=4*PRED_SECONDS+FIT_SETUP_RESERVE+max(0.,elapsed-stages)
|
||||
return prep.first_epoch_gate(candidate,receipts,other_unsettled=0.,approved_greif_cap=CAP,remaining_overhead=overhead)
|
||||
|
||||
|
||||
def run(c, auth, phase_booked):
|
||||
with (ROOT/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (ROOT/'started.json').exists() or any(ROOT.glob('fold-*')):
|
||||
raise FileExistsError('The combined attempt already exists')
|
||||
began=time.monotonic()
|
||||
save(ROOT/'started.json',dict(started_monotonic=began,unix=time.time(),phase_booked=phase_booked,
|
||||
greif_reserved_gpu_seconds=CAP,stage_authorization_sha256=sha(ROOT/'stage-authorization.json')))
|
||||
deadline=began+CAP
|
||||
fit_deadline=deadline-4*PRED_SECONDS
|
||||
status,trigger='STOPPED',None
|
||||
|
||||
def stop(number,_frame):
|
||||
raise KeyboardInterrupt(signal.Signals(number).name)
|
||||
|
||||
handler=signal.signal(signal.SIGTERM,stop)
|
||||
try:
|
||||
first={}
|
||||
for fold in (0,1):
|
||||
result,charge=run_stage(c,auth,fold,0,'first',min(fit_deadline,time.monotonic()+FIRST_STAGE_MAX))
|
||||
first[str(fold)]=dict(fold=fold,seed=0,epoch=1,learning_cases=result['learning_cases'],
|
||||
learning_nodes=result['learning_nodes'],heldout_target_reads=0,
|
||||
epoch_seconds=result['first_epoch_seconds'],booked_gpu_seconds=charge)
|
||||
gate=cost_gate(c,first,phase_booked,time.monotonic()-began)
|
||||
save(ROOT/'first-epoch-cost-gate.json',dict(gate,measurements=first))
|
||||
if not gate['release_epoch2']:
|
||||
raise RuntimeError(gate['status'])
|
||||
models=[]
|
||||
for fold,seed in JOBS:
|
||||
result,_=run_stage(c,auth,fold,seed,'fit',fit_deadline)
|
||||
models.extend(files_for(result))
|
||||
save(ROOT/'models-frozen.json',models)
|
||||
predictions=[]
|
||||
for fold,seed in JOBS:
|
||||
result,_=run_stage(c,auth,fold,seed,'predict',min(deadline,time.monotonic()+PRED_SECONDS))
|
||||
predictions.extend(files_for(result))
|
||||
save(ROOT/'predictions-frozen.json',predictions)
|
||||
status='PREDICTIONS_FROZEN_NO_TARGETS'
|
||||
except (Exception,KeyboardInterrupt) as exc:
|
||||
trigger=str(exc) or type(exc).__name__
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM,handler)
|
||||
charged=time.monotonic()-began
|
||||
if charged>CAP and status!='STOPPED':
|
||||
status,trigger='STOPPED','TOTAL_GPU_CAP'
|
||||
save(ROOT/'complete.json',dict(status=status,trigger=trigger,charged_gpu_seconds=charged,
|
||||
contract_sha256=sha(ROOT/'contract.json'),test_targets_read=False,no_retry=True))
|
||||
return 0 if status=='PREDICTIONS_FROZEN_NO_TARGETS' else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action',choices=('run','pack','score'))
|
||||
args=parser.parse_args()
|
||||
c=contract()
|
||||
auth,booked=authorize(c)
|
||||
if args.action=='run':
|
||||
return run(c,auth,booked)
|
||||
from track_b_gino_greif_custody import pack, score
|
||||
return pack(c) if args.action=='pack' else score(c)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,228 @@
|
||||
"""One guarded Greif job. Only learning tensors are opened by the fit paths."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from phase2_h2_data import cached, standardization
|
||||
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
|
||||
from phase2_residual_math import decode_seconds, residual_target
|
||||
from s2_custody_common import prediction_inputs
|
||||
import track_b_gino_formal_worker as release
|
||||
import track_b_gino_greif_entry as entry
|
||||
from track_b_gino_greif_adapter import GreifGINO, stencil
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
from track_b_gino_shapenet_worker import cuda_peaks, test_environment
|
||||
|
||||
|
||||
def fold_view(c, fold):
|
||||
from track_b_gino_greif_prepare import PILOT, PINS
|
||||
|
||||
pilot=read(checked(PILOT,PINS[str(PILOT)]))
|
||||
row=pilot['folds'][fold]
|
||||
fixed=c['folds'][str(fold)]
|
||||
if any(row[role]!=cases for role,cases in fixed['parts'].items()) or row['clock_sha256']!=fixed['clock_sha256']:
|
||||
raise ValueError('Learning, validation and test roles or original clock differ')
|
||||
return dict(pilot,folds=[row])
|
||||
|
||||
|
||||
def model_input(pilot, case, stats, role):
|
||||
f=pilot['folds'][0]
|
||||
if role not in ('learning','test') or case not in f[role]:
|
||||
raise PermissionError('Original fold role required before reading inputs')
|
||||
pin=pilot['geometry'][case]
|
||||
with np.load(checked(pin['path'],pin['sha256']),allow_pickle=False) as archive:
|
||||
points,raw,mass=(archive[k].copy() for k in ('points','raw_ids','mass'))
|
||||
if len(raw)!=pilot['parts'][case]['nodes'] or not np.array_equal(raw,cached(pilot,case,'raw_ids')) or not np.array_equal(mass,cached(pilot,case,'mass')):
|
||||
raise ValueError('Original full node identity differs')
|
||||
feat=np.asarray(cached(pilot,case,'features'),np.float64)
|
||||
x=((feat-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
||||
fvec=cached(pilot,case,'Vf')
|
||||
if x.shape!=(len(raw),19) or not np.isfinite(x).all() or fvec.dtype!=np.float64 or fvec.shape!=(len(raw),):
|
||||
raise ValueError('Original nineteen columns or FP64 clock differ')
|
||||
corners,weights=stencil(points,raw)
|
||||
args=(torch.from_numpy(x),torch.from_numpy(mass.astype(np.float32)),corners,weights)
|
||||
return args,raw,fvec
|
||||
|
||||
|
||||
def checkpoint(path, model, opt, epoch, fold, seed, stats):
|
||||
temp=path.with_suffix('.tmp')
|
||||
with temp.open('xb') as stream:
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),epochs=epoch,fold=fold,seed=seed,
|
||||
stats=stats,torch_cpu_rng=torch.get_rng_state(),torch_cuda_rng=torch.cuda.get_rng_state(),
|
||||
numpy_rng=np.random.get_state(),python_rng=random.getstate(),
|
||||
contract_sha256=sha(entry.ROOT/'contract.json')),stream)
|
||||
temp.rename(path)
|
||||
|
||||
|
||||
def restore(artifact, fold, seed, epochs):
|
||||
state=torch.load(checked(artifact['path'],artifact['sha256']),map_location='cpu')
|
||||
if (state['fold'],state['seed'],state['epochs'],state['contract_sha256']) != (
|
||||
fold,seed,epochs,sha(entry.ROOT/'contract.json')):
|
||||
raise ValueError('Checkpoint identity differs')
|
||||
return state
|
||||
|
||||
|
||||
def train(c, fold, seed, phase, root, records):
|
||||
p=fold_view(c,fold)
|
||||
f=p['folds'][0]
|
||||
rng_seed=seed
|
||||
random.seed(rng_seed)
|
||||
np.random.seed(rng_seed)
|
||||
torch.manual_seed(rng_seed)
|
||||
torch.cuda.manual_seed_all(rng_seed)
|
||||
model=GreifGINO().cuda()
|
||||
opt=optimizer(model)
|
||||
start, end=1,c['epochs']
|
||||
if phase=='first':
|
||||
if seed!=0:
|
||||
raise PermissionError('First full epoch is seed 0 only')
|
||||
end=1
|
||||
normalization=root/'norm'
|
||||
normalization.mkdir(exist_ok=False)
|
||||
stats=standardization(p,normalization)
|
||||
else:
|
||||
cost=read(entry.ROOT/'first-epoch-cost-gate.json')
|
||||
if not cost['release_epoch2']:
|
||||
raise PermissionError('Both measured full epochs must pass the cost gate')
|
||||
earlier=entry.stage_terminal(fold,0,'first')['files'][0]
|
||||
state=restore(earlier,fold,0,1)
|
||||
stats=state['stats']
|
||||
if seed==0:
|
||||
start=2
|
||||
model.load_state_dict(state['model'],strict=True)
|
||||
opt.load_state_dict(state['optimizer'])
|
||||
torch.set_rng_state(state['torch_cpu_rng'])
|
||||
torch.cuda.set_rng_state(state['torch_cuda_rng'])
|
||||
np.random.set_state(state['numpy_rng'])
|
||||
random.setstate(state['python_rng'])
|
||||
del state
|
||||
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
|
||||
raise ValueError('Only the learning normalization of this fold is valid')
|
||||
ncases,nnodes=len(f['learning']),f['learning_nodes']
|
||||
duration=None
|
||||
with (root/'learning-access.jsonl').open('x') as access:
|
||||
for epoch in range(start,end+1):
|
||||
started=time.monotonic()
|
||||
model.train()
|
||||
for group in opt.param_groups:
|
||||
group['lr']=lr_epoch(epoch)
|
||||
seen=0
|
||||
order=np.random.default_rng(np.random.SeedSequence([seed,epoch])).permutation(ncases)
|
||||
for index in order:
|
||||
case=f['learning'][int(index)]
|
||||
args,raw,fvec=model_input(p,case,stats,'learning')
|
||||
access.write(json.dumps(dict(case=case,fold=fold,seed=seed,epoch=epoch,role='learning',
|
||||
sha256=p['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
access.flush()
|
||||
target=residual_target(cached(p,case,'target'),fvec,f['c'],fit_dtype='float32')
|
||||
opt.zero_grad(set_to_none=True)
|
||||
prediction=model(*(a.cuda() for a in args))
|
||||
loss=weighted_loss(prediction,torch.from_numpy(target).cuda(),ncases,len(raw),nnodes)
|
||||
if not bool(torch.isfinite(loss)):
|
||||
raise ValueError('Nonfinite full-node H2-weighted loss')
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
|
||||
opt.step()
|
||||
if any(not bool(torch.isfinite(value).all()) for value in model.parameters()):
|
||||
raise ValueError('Nonfinite model weight after update')
|
||||
torch.cuda.synchronize()
|
||||
seen+=len(raw)
|
||||
del prediction,loss,args,raw,fvec,target
|
||||
if seen!=nnodes:
|
||||
raise ValueError('Incomplete full-node learning epoch')
|
||||
duration=time.monotonic()-started
|
||||
save(records/'progress.json',dict(stage='EPOCH_COMPLETE',epoch=epoch,
|
||||
learning_cases=ncases,learning_nodes=seen,seconds=duration,**cuda_peaks(torch)))
|
||||
output=root/f'epoch-{end}.pt'
|
||||
checkpoint(output,model,opt,end,fold,seed,stats)
|
||||
return dict(epochs=end,learning_cases=ncases,learning_nodes=nnodes,
|
||||
first_epoch_seconds=duration if phase=='first' else None,files=[
|
||||
dict(path=str(output),sha256=sha(output),fold=fold,seed=seed)])
|
||||
|
||||
|
||||
def predict(c, fold, seed, root, records):
|
||||
from track_b_gino_greif_prepare import QUERY, PINS
|
||||
|
||||
p=fold_view(c,fold)
|
||||
f=p['folds'][0]
|
||||
models=read(entry.ROOT/'models-frozen.json')
|
||||
by={(r['fold'],r['seed']):r for r in models}
|
||||
if len(models)!=4 or set(by)!=set(entry.JOBS):
|
||||
raise PermissionError('Predict only after four frozen endpoints')
|
||||
state=restore(by[fold,seed],fold,seed,c['epochs'])
|
||||
stats=state['stats']
|
||||
model=GreifGINO().cuda()
|
||||
model.load_state_dict(state['model'],strict=True)
|
||||
model.eval()
|
||||
del state
|
||||
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
|
||||
raise ValueError('Original fold normalization differs')
|
||||
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
|
||||
entries=[]
|
||||
with torch.no_grad():
|
||||
for case in f['test']:
|
||||
args,raw,fvec=model_input(p,case,stats,'test')
|
||||
q=prediction_inputs(query[case])
|
||||
ix=np.searchsorted(raw,q['raw_ids'])
|
||||
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
|
||||
raise ValueError('Frozen PPS query-to-node map differs')
|
||||
residual=model(*(a.cuda() for a in args)).cpu().numpy()
|
||||
seconds=decode_seconds(residual,fvec,f['c'],p['parts'][case]['t_set'])
|
||||
item=root/(case+'.npz')
|
||||
with item.open('xb') as file:
|
||||
np.savez_compressed(file,seconds=seconds,raw_ids=raw,query_raw_ids=q['raw_ids'],
|
||||
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg'])
|
||||
entries.append(dict(case=case,fold=fold,seed=seed,family=query[case]['component_id'],
|
||||
path=str(item),sha256=sha(item)))
|
||||
save(records/'progress.json',dict(stage='PREDICT_NO_TARGETS',cases=len(entries),**cuda_peaks(torch)))
|
||||
return dict(files=entries)
|
||||
|
||||
|
||||
def run(fold,seed,phase):
|
||||
c=entry.contract()
|
||||
auth,_=entry.authorize(c)
|
||||
root=entry.path(fold,seed,phase)
|
||||
ticket=read(root/'dispatch.json')
|
||||
parent=os.getppid()
|
||||
if (ticket['coordinator']!=process_identity(parent) or (ticket['fold'],ticket['seed'],ticket['phase'])!=(fold,seed,phase)
|
||||
or ticket['contract_sha256']!=sha(entry.ROOT/'contract.json')
|
||||
or time.monotonic()>=ticket['start_by_monotonic'] or time.time()>=auth['dispatch_before_unix']
|
||||
or str(Path(sys.executable).absolute())!=c['environment']['python']):
|
||||
raise PermissionError('Worker not attached to the reviewed guard')
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0) or os.getppid()!=parent:
|
||||
raise RuntimeError('Could not bind worker lifetime to the guard')
|
||||
test_environment(c)
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
device=entry.DEVICES[int(ticket['ordinal'])]
|
||||
binder=FunctionType(release.bind_device.__code__,dict(release.bind_device.__globals__,
|
||||
formal=SimpleNamespace(DEVICES={str(seed):device})),release.bind_device.__name__)
|
||||
binder(root,c,ticket,torch)
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
records=root/'worker-records'
|
||||
records.mkdir(exist_ok=False)
|
||||
result=predict(c,fold,seed,root,records) if phase=='predict' else train(c,fold,seed,phase,root,records)
|
||||
save(records/'result.json',dict(result,status='COMPLETE',fold=fold,seed=seed,phase=phase,
|
||||
test_targets_read=False,**cuda_peaks(torch)))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('fold',type=int,choices=(0,1))
|
||||
parser.add_argument('seed',type=int,choices=(0,1))
|
||||
parser.add_argument('phase',choices=('first','fit','predict'))
|
||||
args=parser.parse_args()
|
||||
run(args.fold,args.seed,args.phase)
|
||||
Reference in new issue
Block a user