Bind one Greif training, custody and paired scoring entry for review

This commit is contained in:
NewMeltAssist committed 2026-11-19 19:31:04 +08:00
1 parent 8810276beb
commit 3c82c074e8
9 files changed
+3419

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,73 @@
{
"status": "PASS",
"tests": 7,
"actual_gpu_queries": 0,
"real_target_arrays_read": 0,
"scenarios": {
"nominal": {
"status": "PREDICTIONS_FROZEN_NO_TARGETS",
"trigger": null,
"stages": [
[
0,
0,
"first"
],
[
1,
0,
"first"
],
[
0,
0,
"fit"
],
[
0,
1,
"fit"
],
[
1,
0,
"fit"
],
[
1,
1,
"fit"
],
[
0,
0,
"predict"
],
[
0,
1,
"predict"
],
[
1,
0,
"predict"
],
[
1,
1,
"predict"
]
],
"cpu_child_spawns": 10
},
"busy": {
"status": "STOPPED",
"trigger": "0/0/first IDLE_DEADLINE",
"stages": [],
"cpu_child_spawns": 0
}
},
"contract_sha256": "f56e0d1a0e64fb46f2af6a731730308856083bdd18cfe030ab478b0cd841d539",
"replay_log_sha256": "33f471940854a4ac2a988f0cfd12a857016c8aea97197d2d26ff6ca52e6c7684"
}
@@ -0,0 +1,12 @@
test_actual_checkpoint_preserves_optimizer_for_second_update (test_track_b_gino_greif_combined.Combined) ... ok
test_first_cost_gate_counts_spent_and_blocks_over_budget (test_track_b_gino_greif_combined.Combined) ... ok
test_matching_drive_checksum_is_required (test_track_b_gino_greif_combined.Combined) ... ok
test_no_review_prevents_any_gpu_probe (test_track_b_gino_greif_combined.Combined) ... ok
test_replays_actual_production_guard_and_stage_order (test_track_b_gino_greif_combined.Combined) ... ok
test_scoring_decision_fixed_roster_and_named_hard_intersection (test_track_b_gino_greif_combined.Combined) ... ok
test_two_predictions_share_exactly_one_mock_target_open (test_track_b_gino_greif_combined.Combined) ... ok
----------------------------------------------------------------------
Ran 7 tests in 2.532s
OK
@@ -0,0 +1,17 @@
# GINO-Greif 训练、预测、保管与评分联合预审
此绑定采用已验收的适配器 `01d6581c`。固定 grouped folds 0/1、seeds 0/1、200 轮,以及原 H2 的 19 列、学习集标准化、FP64 `Vf`、折内冻结 `c` 和 PPS 采样。四个模型沿用 H2 的 AdamW、200 轮学习率日程、逐节点等权 MSE 和梯度裁剪。预测输出覆盖全部原节点,再按原始查询评分。 [Claude 裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795087203745169) · [联合合同](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/contract.json)
## 训练与成本
在同一次正式运行内先完成两折 seed 0 的首轮,保存模型、优化器、折内标准化和随机状态。两个实测均覆盖完整学习域才检查成本。通过后两条 seed 0 从第 2 轮续接,seed 1 从第 1 轮训练。首轮各记一次,未执行轮数每折为 399。投影为两折首轮已耗加各自余轮实测乘 1.25,再加未来训练设置 1,200 GPU 秒及四段预测各 1,800 GPU 秒。缺测则无续接,超本项 100,000 GPU 秒或 Phase 3 共 200 GPU 小时即停止。实际阶段账按统一时钟和实际守卫核验。GPU0–2 优先,GPU3 最后且独占。 [入口、成本和阶段门](../tools/track_b_gino_greif_entry.py)
## 保管与评分
同一入口的 `run` 完成训练和无目标预测,`pack` 将四份最终权重和 424 份预测归档。批准后,执行方用 Drive 连接器上传原包到用户的私有项目目录,再读取文件 ID、父目录、字节数和 SHA-256,核实项目内权限并登记。Git 只收小索引。只有完成终态、四模型与全预测哈希以及 Drive 回读均匹配时,`score` 才用 CPU 打开留出数据。每件实际目标只打开一次,供两种子共享;已有 H2 指标按冻结配对清单复用。 [训练与预测代码](../tools/track_b_gino_greif_worker.py) · [保管与评分代码](../tools/track_b_gino_greif_custody.py)
主裁决固定 212 件、78 家族。每件先在每个模型内等权平均两个种子的时场相对 L2,再在家族内平均,最后家族等权。对 GINO−H2 做 2,000 次配对家族 bootstrap,95% 区间上界低于零认定 GINO 更好,下界高于零认定 H2 更好,含零归为相当。末填、Kendall τ-b 和既有 34 难件名单在两折中的 12 件交集只作描述。主指标缺支持时保留缺项。 [预登记比较](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/comparison-contract.json)
## 当前验收
七项新增 CPU 回放通过。生产守卫用冻结合同顺序跑完两次首轮、四次训练续接和四次预测的合成子进程,忙卡零派发。另验第一轮优化器状态保留、缺测和超帽、Drive 校验、配对判决及单目标读取。真实 GPU 查询、训练、预测、上传和留出读取均为零。独立 PASS、ShapeNet 正式裁定、Yanqi 本项明确启动许可和足额实时 Phase 3 账均待采用。 [CPU 回执](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review/cpu-replay.json)
+201
View File
@@ -0,0 +1,201 @@
"""CPU replay for the reviewed Greif cycle, first-epoch continuity and one-read scorer."""
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import time
import unittest
from unittest.mock import patch
import numpy as np
import torch
from s2_gpu_idle_gate import wait_idle
import track_b_gino_formal_guard as guard
import track_b_gino_greif_custody as custody
import track_b_gino_greif_entry as entry
import track_b_gino_greif_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
EVIDENCE={}
class Combined(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.c=entry.contract()
cls.contract_path=entry.ROOT/'contract.json'
cls.comparison=read(entry.ROOT/'comparison-contract.json')
def test_replays_actual_production_guard_and_stage_order(self):
for busy in (False,True):
with self.subTest(busy=busy), TemporaryDirectory() as directory:
root=Path(directory)
(root/'contract.json').write_bytes(self.contract_path.read_bytes())
save(root/'stage-authorization.json',dict(test=True))
path=root/'track_b_gino_greif_worker.py'
path.write_text(
'import json,sys,pathlib\n'
'root=pathlib.Path(__file__).parent\n'
'fold,seed,phase=sys.argv[1:]\n'
'stage=root/f"fold-{fold}"/f"seed-{seed}"/phase\n'
'p=stage/"worker-records"; p.mkdir()\n'
'spec=json.loads((root/"contract.json").read_text())["folds"][fold]\n'
'obj=dict(learning_cases=spec["counts"]["learning"],learning_nodes=spec["learning_nodes"],'
'first_epoch_seconds=.01,files=[],peak_allocated_bytes=1,peak_reserved_bytes=2)\n'
'(p/"temp").write_text(json.dumps(obj)); (p/"temp").rename(p/"result.json")\n')
observed=[]
elapsed=[0.]
card=entry.DEVICES[0]
def idle(uuid):
return dict(uuid=uuid,idle=True,occupants=[],memory_MiB=0,unix=10.)
def device(uuid):
return dict(idle(uuid),idle=not busy)
def simulated_idle(uuid, limit, record, *, sample):
return wait_idle(uuid,limit,record,sample=sample,clock=lambda:elapsed[0],
sleep=lambda s:elapsed.__setitem__(0,elapsed[0]+s))
def validate(_,fold,seed,phase):
observed.append((fold,seed,phase))
return sha(entry.path(fold,seed,phase)/'worker-records/result.json')
with (patch.object(entry,'ROOT',root),patch.object(entry,'SOURCE',root),
patch.object(entry,'snapshot',side_effect=idle),
patch.object(entry,'validate_result',side_effect=validate),
patch.object(guard,'snapshot',side_effect=device),
patch.object(guard,'wait_idle',side_effect=simulated_idle),
patch.object(guard,'dispatch_environment',
side_effect=lambda u,env:(dict(env,CUDA_VISIBLE_DEVICES='0'),[]))):
result=entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
with self.assertRaises(FileExistsError):
entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
terminal=read(root/'complete.json')
p=root/'fold-0/seed-0/first'
if busy:
self.assertEqual(result,1)
self.assertIn('IDLE_DEADLINE',terminal['trigger'])
self.assertFalse((p/'actual-worker-spawn.json').is_file())
self.assertFalse((root/'predictions-frozen.json').exists())
self.assertEqual(observed,[])
else:
expected=[(0,0,'first'),(1,0,'first')]+[(f,s,p) for p in ('fit','predict') for f,s in entry.JOBS]
self.assertEqual(result,0)
self.assertEqual(observed,expected)
self.assertTrue(read(root/'first-epoch-cost-gate.json')['release_epoch2'])
for f,s,phase in observed:
event=read(root/f'fold-{f}'/f'seed-{s}'/phase/'complete.json')
self.assertEqual(event['physical_uuid'],card)
self.assertEqual([v['event'] for v in event['lifecycle']],['leader_reaped'])
self.assertFalse((root/'score').exists())
EVIDENCE['busy' if busy else 'nominal']=dict(
status=terminal['status'],trigger=terminal['trigger'],stages=observed,
cpu_child_spawns=len(list(root.rglob('actual-worker-spawn.json'))))
def test_first_cost_gate_counts_spent_and_blocks_over_budget(self):
common=dict(seed=0,epoch=1,heldout_target_reads=0,booked_gpu_seconds=400.)
p={k:dict(common,fold=int(k),epoch_seconds=70.,learning_nodes=r['learning_nodes'],
learning_cases=r['counts']['learning']) for k,r in self.c['folds'].items()}
gate=entry.cost_gate(self.c,p,495434.9566447616,820.)
self.assertTrue(gate['release_epoch2'])
self.assertEqual(gate['charged_first_epochs_and_setup_gpu_seconds'],800.)
self.assertEqual(gate['remaining_train_gpu_seconds'],(70+70)*399)
self.assertFalse(entry.cost_gate(self.c,{'0':p['0']},495434.9566447616,820.)['release_epoch2'])
p['1']['epoch_seconds']=185.
self.assertEqual(entry.cost_gate(self.c,p,495434.9566447616,820.)['status'],'STOP_PROJECTED_CAP')
p['1']['epoch_seconds']=70.
self.assertFalse(entry.cost_gate(self.c,p,680000.,820.)['release_epoch2'])
def test_actual_checkpoint_preserves_optimizer_for_second_update(self):
torch.manual_seed(7)
model=torch.nn.Linear(2,1)
opt=torch.optim.AdamW(model.parameters())
x=torch.tensor([[3.,7.]])
def step(m,o):
o.zero_grad()
m(x).square().sum().backward()
o.step()
step(model,opt)
with TemporaryDirectory() as directory, patch.object(torch.cuda,'get_rng_state',return_value=torch.get_rng_state()):
path=Path(directory)/'epoch-1.pt'
worker.checkpoint(path,model,opt,1,0,0,{'only_learning':True})
saved=worker.restore(dict(path=str(path),sha256=sha(path)),0,0,1)
copy=torch.nn.Linear(2,1)
copy.load_state_dict(saved['model'])
resumed=torch.optim.AdamW(copy.parameters())
resumed.load_state_dict(saved['optimizer'])
step(model,opt)
step(copy,resumed)
self.assertTrue(all(torch.equal(a,b) for a,b in zip(model.parameters(),copy.parameters())))
def test_scoring_decision_fixed_roster_and_named_hard_intersection(self):
def metrics(x):
return {k:dict(value=x,reason=None) for k in custody.METRICS}
for g,h,decision in ((.06,.07,'GINO_BETTER'),(.07,.07,'COMPARABLE'),(.08,.07,'H2_BETTER')):
rows=[dict(r,gino=[metrics(g+.001),metrics(g-.001)],h2=[metrics(h),metrics(h)])
for r in self.comparison['roster']]
result=custody.summary(rows,self.comparison)
self.assertEqual(result['field']['decision'],decision)
self.assertEqual(result['field']['components'],78)
self.assertEqual(result['hard_subset']['frozen_total'],34)
rows[0]['gino'][0]['relative_l2']['value']=None
self.assertEqual(custody.summary(rows,self.comparison)['field']['decision'],'INCOMPLETE_PAIRED_SUPPORT')
with self.assertRaises(ValueError):
custody.summary(rows[:-1],self.comparison)
def test_matching_drive_checksum_is_required(self):
with TemporaryDirectory() as directory, patch.object(entry,'ROOT',Path(directory)):
root=Path(directory)
for n in ('contract.json','models-frozen.json','predictions-frozen.json'):
save(root/n,[])
path=root/'gino-greif-models-and-predictions.zip'
path.write_bytes(b'fixture')
bundle=dict(path=str(path),sha256=sha(path),bytes=path.stat().st_size,contract_sha256=sha(root/'contract.json'),
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json'))
save(root/'drive-upload-index.json',bundle)
auth=dict(private_drive_folder_id='folder-fixture')
receipt=dict(sha256Checksum='wrong',size=str(bundle['bytes']),parent_id='folder-fixture',
file_id='fixture',project_private=True,contract_sha256=bundle['contract_sha256'])
save(root/'drive-readback.json',receipt)
with self.assertRaises(PermissionError):
custody.verify_drive(auth)
save(root/'drive-readback.json',dict(receipt,sha256Checksum=sha(path)))
self.assertEqual(custody.verify_drive(auth)['file_id'],'fixture')
def test_two_predictions_share_exactly_one_mock_target_open(self):
with TemporaryDirectory() as directory:
root=Path(directory)
raw=np.array([20,24,28])
coords=np.array([[0.,0.,0.],[1.,0.,0.],[2.,0.,0.]])
q=dict(raw_ids=raw,counts=np.array([1,1,1]),near_wall=np.array([True,False,False]),coordinates_over_Rg=coords)
row=dict(fold=0,case='0003_fixture',family='test-family')
forecasts={}
for seed in (0,1):
p=root/(str(seed)+'.npz')
np.savez(p,seconds=np.array([1.,2.,4.]),raw_ids=raw,query_raw_ids=raw,
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=coords)
forecasts[0,seed,row['case']]=dict(family=row['family'],path=str(p),sha256=sha(p))
pilot=dict(parts={row['case']:dict(output_sha256={'seconds.npy':'fixture'})})
ref={row['case']:dict(fold=0,component_id=row['family'],seed0={},seed1={})}
with ((root/'log').open('x') as log, patch.object(custody,'prediction_inputs',return_value=q),
patch.object(custody,'cached',return_value=raw),
patch.object(custody,'target_array',return_value=np.array([1.,2.,3.])) as target):
result=custody.one_case(pilot,{},row,forecasts,ref,log)
target.assert_called_once_with(pilot,row['case'])
self.assertEqual(len(result['gino']),2)
self.assertEqual(len((root/'log').read_text().splitlines()),1)
def test_no_review_prevents_any_gpu_probe(self):
with patch.object(entry,'snapshot') as probe:
with self.assertRaises(FileNotFoundError):
entry.authorize(self.c)
probe.assert_not_called()
if __name__=='__main__':
unittest.main()
+205
View File
@@ -0,0 +1,205 @@
"""Drive-backed Greif safekeeping and one guarded custody read per test case."""
from collections import defaultdict
import hashlib
import io
import json
import os
from pathlib import Path
import resource
import time
from zipfile import ZIP_STORED, ZipFile
import numpy as np
from phase2_h2_data import cached
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
import track_b_gino_greif_entry as entry
from track_b_gino_greif_prepare import PILOT, QUERY, PINS
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
METRICS=('relative_l2','last_location','kendall_tau_b')
def frozen(c):
root=entry.ROOT
terminal=read(root/'complete.json')
if (terminal['status'],terminal['contract_sha256'],terminal['test_targets_read']) != (
'PREDICTIONS_FROZEN_NO_TARGETS',sha(root/'contract.json'),False) or not 0<terminal['charged_gpu_seconds']<=c['budgets']['gpu_cap']:
raise PermissionError('A capped completed prediction terminal must precede custody')
models=[]
predictions=[]
for fold,seed in entry.JOBS:
models.extend(entry.stage_terminal(fold,seed,'fit')['files'])
predictions.extend(entry.stage_terminal(fold,seed,'predict')['files'])
if models!=read(root/'models-frozen.json') or predictions!=read(root/'predictions-frozen.json'):
raise ValueError('Final sealed indexes differ from accepted terminals')
if len(models)!=4 or len(predictions)!=424 or len({r['path'] for r in models+predictions})!=428:
raise ValueError('Four final weights and 424 forecasts are required')
for item in models+predictions:
checked(item['path'],item['sha256'])
if not Path(item['path']).resolve().is_relative_to(root):
raise ValueError('Artifact path leaves the frozen run root')
return models,predictions
def pack(c):
root=entry.ROOT
models,predictions=frozen(c)
dest=root/'gino-greif-models-and-predictions.zip'
with ZipFile(dest,'x',compression=ZIP_STORED) as archive:
for item in models+predictions:
archive.write(item['path'],Path(item['path']).relative_to(root))
for name in ('models-frozen.json','predictions-frozen.json','contract.json'):
archive.write(root/name,name)
with ZipFile(dest,'r') as archive:
for item in models+predictions:
key=str(Path(item['path']).relative_to(root))
with archive.open(key) as stream:
actual=hashlib.sha256()
for data in iter(lambda:stream.read(1024**2),b''):
actual.update(data)
if actual.hexdigest()!=item['sha256']:
raise ValueError('Archived bytes differ from the frozen file index')
save(root/'drive-upload-index.json',dict(status='READY_FOR_PRIVATE_PROJECT_DRIVE',path=str(dest),
bytes=dest.stat().st_size,sha256=sha(dest),contract_sha256=sha(root/'contract.json'),
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json')))
return 0
def verify_drive(auth):
root=entry.ROOT
bundle=read(root/'drive-upload-index.json')
path=checked(bundle['path'],bundle['sha256'])
if (path!=root/'gino-greif-models-and-predictions.zip'
or path.stat().st_size!=bundle['bytes']
or bundle['contract_sha256']!=sha(root/'contract.json')
or bundle['model_index_sha256']!=sha(root/'models-frozen.json')
or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')):
raise ValueError('Local archive no longer matches the fixed models and forecasts')
receipt=read(root/'drive-readback.json')
# Drive file metadata, recorded by the authorized connector from the private project folder.
if (receipt.get('sha256Checksum'),receipt.get('size'),receipt.get('parent_id')) != (
bundle['sha256'],str(bundle['bytes']),auth['private_drive_folder_id']):
raise PermissionError('Private Drive readback SHA, size and folder are required before scoring')
if (not receipt.get('file_id') or receipt.get('project_private') is not True
or receipt.get('contract_sha256')!=bundle['contract_sha256']):
raise PermissionError('Private Drive identity required before opening targets')
return receipt
def bootstrap(rows, metric, fixed_support=None):
values=defaultdict(list)
unavailable=[]
for row in rows:
scores=[row[arm][seed][metric]['value'] for arm in ('gino','h2') for seed in (0,1)]
if any(v is None or not np.isfinite(v) for v in scores):
unavailable.append(row['case'])
continue
values[row['family']].append(((scores[0]+scores[1])/2,(scores[2]+scores[3])/2))
absent=set(unavailable)
rejected={r['family'] for r in rows if r['case'] in absent}
family=np.array([np.mean(values[k],axis=0) for k in sorted(set(values)-rejected)])
support=dict(parts=len(rows),components=len(family),undefined_cases=unavailable)
if not len(family) or (fixed_support is not None and (len(rows),len(family))!=fixed_support):
return dict(**support,decision='INCOMPLETE_PAIRED_SUPPORT',ci95=None)
delta=family[:,0]-family[:,1]
rng=np.random.default_rng(2026111408)
draw=rng.integers(len(family),size=(2000,len(family)))
low,high=map(float,np.percentile(delta[draw].mean(axis=1),[2.5,97.5]))
return dict(**support,gino=float(family[:,0].mean()),h2=float(family[:,1].mean()),delta=float(delta.mean()),
ci95=[low,high],replicates=2000,seed=2026111408)
def summary(rows, compare):
if sorted((r['fold'],r['case'],r['family']) for r in rows)!=sorted(
(r['fold'],r['case'],r['family']) for r in compare['roster']):
raise ValueError('Exact two-fold paired roster required')
primary=bootstrap(rows,'relative_l2',fixed_support=(212,78))
if primary['ci95'] is not None:
low,high=primary['ci95']
primary['decision']='GINO_BETTER' if high<0 else 'H2_BETTER' if low>0 else 'COMPARABLE'
hard=[r for r in rows if r['case'] in compare['hard34']]
return dict(field=primary,secondary={metric:bootstrap(rows,metric) for metric in METRICS[1:]},
hard_subset=dict(frozen_total=34,two_fold_cases=[r['case'] for r in hard],
descriptive={metric:bootstrap(hard,metric) for metric in METRICS}))
def target_array(pilot,case):
item=pilot['parts'][case]
file=BASE/item['output']/'seconds.npy'
blob=file.read_bytes()
if hashlib.sha256(blob).hexdigest()!=item['output_sha256']['seconds.npy']:
raise ValueError('Pinned held-out bytes differ')
return np.load(io.BytesIO(blob),allow_pickle=False)
def one_case(pilot, meta, row, forecasts, reference, log):
case=row['case']
fold=row['fold']
q=prediction_inputs(meta)
raw=cached(pilot,case,'raw_ids')
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
raise ValueError('Frozen PPS coordinates differ')
new=[]
for seed in (0,1):
f=forecasts[fold,seed,case]
if f['family']!=row['family']:
raise ValueError('Predicted family differs')
with np.load(checked(f['path'],f['sha256']),allow_pickle=False) as z:
seconds=z['seconds']
if (seconds.dtype!=np.float64 or seconds.shape!=raw.shape or not np.isfinite(seconds).all()
or not np.array_equal(raw,z['raw_ids'])
or any(not np.array_equal(q[name],z['query_raw_ids' if name=='raw_ids' else name])
for name in ('counts','near_wall','coordinates_over_Rg','raw_ids'))):
raise ValueError('Frozen full node or PPS support differs')
new.append(seconds[ix])
old=reference[case]
if (old['fold'],old['component_id'])!=(fold,row['family']):
raise ValueError('H2 score pairing differs')
log.write(json.dumps(dict(case=case,fold=fold,role='test',sha256=pilot['parts'][case]['output_sha256']['seconds.npy']))+'\n')
log.flush()
truth=target_array(pilot,case)
if truth.shape!=raw.shape:
raise ValueError('Original held-out node support differs')
gino=[part_metrics(p,truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],
int(case.split('_')[0])) for p in new]
return dict(row,gino=gino,h2=[old['seed0'],old['seed1']])
def score(c):
auth,_=entry.authorize(c)
if os.environ.get('CUDA_VISIBLE_DEVICES')!='':
raise PermissionError('Custody scoring must run with CUDA hidden')
ceiling=resource.getrlimit(resource.RLIMIT_CPU)[1]
limit=c['budgets']['score_cpu_seconds']
if ceiling!=resource.RLIM_INFINITY:
limit=min(limit,ceiling)
resource.setrlimit(resource.RLIMIT_CPU,(limit,limit))
_,predictions=frozen(c)
drive=verify_drive(auth)
compare=read(checked(c['comparison_contract'],c['comparison_contract_sha256']))
rosters={(r['fold'],r['case']) for r in compare['roster']}
new={(r['fold'],r['seed'],r['case']):r for r in predictions}
if set(new)!={(fold,seed,case) for fold,case in rosters for seed in (0,1)}:
raise ValueError('Frozen four-set test support differs')
prior=read(checked(compare['h2_score']['path'],compare['h2_score']['sha256']))
reference={r['case']:r for r in prior['rows']}
if prior['status']!='COMPLETE' or len(reference)!=552 or not {r['case'] for r in compare['roster']}<=set(reference):
raise ValueError('H2 frozen score support differs')
pilot=read(checked(PILOT,PINS[str(PILOT)]))
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
out=entry.ROOT/'score'
out.mkdir(exist_ok=False)
save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'],
manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),before_test_target_reads=True))
results=[]
with (out/'heldout-target-access.jsonl').open('x') as log:
for row in compare['roster']:
results.append(one_case(pilot,query[row['case']],row,new,reference,log))
save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results,
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id']))
return 0
+259
View File
@@ -0,0 +1,259 @@
"""Reviewed GINO-Greif training, predictions and Drive-backed custody entry."""
import argparse
import fcntl
import json
import math
from pathlib import Path
import signal
import time
from phase2_gate_outcome_review import validate_review
from s2_gpu_idle_gate import snapshot
import track_b_gino_greif_prepare as prep
from track_b_gino_formal_guard import guard
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = prep.ROOT / 'combined-review'
SOURCE = Path(__file__).resolve().parent
ADAPTER = prep.REVIEW_ROOT / 'preparation-binding.json'
ADAPTER_SHA = '01d6581c33a6c26d1965acb13248bdb741de892e5929fff08a7e4af431f00d0b'
H2_SCORE = prep.H2_REFERENCE[1] / 'score/result.json'
H2_SCORE_SHA = 'd259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4'
HARD = BASE / 'artifacts/phase3-h2-shared-miss-attributes-2026-11-19/by-part.jsonl'
DEVICES = ('GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a')
JOBS = ((0,0),(0,1),(1,0),(1,1))
CAP = 100000
PRED_SECONDS = 1800
FIT_SETUP_RESERVE = 1200
FIRST_STAGE_MAX = 3600
def comparison(candidate):
rows = [json.loads(line) for line in HARD.read_text().splitlines()]
hard = [r['case'] for r in rows if r['shared_miss']]
if len(hard) != 34 or len(set(hard)) != 34:
raise ValueError('The original named hard cohort is required')
query = read(checked(prep.QUERY, prep.PINS[str(prep.QUERY)]))
family = {r['case']:r['component_id'] for r in query['rows'] if r['included']}
roster = [dict(fold=int(fold), case=case, family=family[case])
for fold,data in candidate['folds'].items() for case in data['parts']['test']]
if len({r['case'] for r in roster}) != 212 or len({r['family'] for r in roster}) != 78:
raise ValueError('The fixed 212-part / 78-family support is required')
return dict(roster=roster, hard34=hard, frozen_hard_source=dict(path=str(HARD),sha256=sha(HARD)),
h2_score=dict(path=str(H2_SCORE),sha256=H2_SCORE_SHA), method='Paired component bootstrap of GINO minus H2',
metric='field relative L2; mean of seed0 and seed1 per arm, then case and component means',
confidence=.95, replicates=2000, bootstrap_seed=2026111408,
gates={'high_below_zero':'GINO_BETTER','contains_zero':'COMPARABLE','low_above_zero':'H2_BETTER'},
secondary=['last_location/Rg','Kendall tau-b','intersection of named hard34 and two-fold test'],
custody='each of the 212 original targets once for both new predictions, after all four checkpoints and predictions frozen')
def expected_contract():
c = read(checked(ADAPTER, ADAPTER_SHA))
for path,h in c['code_sha256'].items():
checked(path,h)
base = prep.reference.contract()
comparison_path = ROOT / 'comparison-contract.json'
if read(comparison_path) != comparison(c):
raise ValueError('Pre-registered Greif comparison differs')
code=dict(base['code_sha256'],**c['code_sha256'])
for name in ('track_b_gino_greif_entry.py','track_b_gino_greif_worker.py','track_b_gino_greif_custody.py',
'phase2_h2_model.py','phase2_h2_data.py','s2_custody_common.py','s2_custody_statistics.py'):
code[str(SOURCE/name)] = sha(SOURCE/name)
return dict(handoff='h-08c2a2a5',mode='gino-greif-combined',adapter_sha256=ADAPTER_SHA,
comparison_contract=str(comparison_path),comparison_contract_sha256=sha(comparison_path),code_sha256=code,
folds=c['folds'],jobs=[list(x) for x in JOBS],epochs=c['epochs'],environment=base['environment'],
fixed_gino_reference=str(prep.reference.ROOT),phase3_anchor=c['projection'],limits=base['limits'],
device_order=list(DEVICES),gpu3_single_task=True,
budgets=dict(gpu_cap=CAP,phase3_cap=prep.CAP,max_first_stage=FIRST_STAGE_MAX,
per_prediction=PRED_SECONDS,train_setup_reserve=FIT_SETUP_RESERVE,score_cpu_seconds=3600),
run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits',
'freeze models','form four target-free prediction sets','freeze predictions',
'verified private Drive transfer','one controlled custody pass'],
drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only'},
no_retry=True)
def contract():
c = read(ROOT/'contract.json')
if c != expected_contract():
raise PermissionError('Exact reviewed Greif contract differs')
return c
def authorize(c):
validate_review(ROOT)
a=read(ROOT/'stage-authorization.json')
if (a.get('decision'),a.get('mode'),a.get('contract_sha256'),a.get('review_sha256')) != (
'APPROVED',c['mode'],sha(ROOT/'contract.json'),sha(ROOT/'review.json')):
raise PermissionError('Exact review plus separate stage authorization required')
if (not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id')
or not a.get('private_drive_folder_id')):
raise PermissionError('Direct user launch and ShapeNet verdict are required')
verdict=read(checked(a['shape_verdict']['path'],a['shape_verdict']['sha256']))
if (verdict['contract_sha256']!=prep.reference.sha(prep.reference.ROOT/'contract.json')
or verdict['decision'] not in ('REPRODUCED','NOT_REPRODUCED')
or verdict['claude_ruling_id']!=a['claude_shape_verdict_id']):
raise PermissionError('Exact final ShapeNet verdict receipt required')
latest=read(checked(a['phase3_ledger']['path'],a['phase3_ledger']['sha256']))
stamp=read(ROOT/'started.json')['unix'] if (ROOT/'started.json').exists() else time.time()
# Charge each currently active program at its full un-settled reserve.
keys={'earlier_phase3_settled','gino_old_measurement','gino_formal','h2_pressure','other'}
figures=latest['gpu_seconds']
p=c['phase3_anchor']
if (latest['phase3_cap_gpu_seconds']!=prep.CAP or not 0<=stamp-latest['checked_unix']<=900
or set(figures)!=keys
or figures['earlier_phase3_settled']<p['settled_phase3_through_1b_gpu_seconds']
or figures['gino_old_measurement']!=p['gino_measurement_once_gpu_seconds']
or figures['gino_formal']<p['gino_shapenet_formal_reserve_gpu_seconds']
or figures['h2_pressure']<p['h2_pressure_unsettled_max_gpu_seconds']
or any(type(x) not in (int,float) or not math.isfinite(x) or x<0 for x in figures.values())
or sum(figures.values())+CAP>prep.CAP
or type(a['dispatch_before_unix']) not in (float,int) or not math.isfinite(a['dispatch_before_unix'])):
raise PermissionError('Live phase-3 ledger cannot book the declared Greif cap')
return a, sum(figures.values())
def path(fold, seed, phase):
return ROOT/f'fold-{fold}'/f'seed-{seed}'/phase
def stage_terminal(fold, seed, phase):
root = path(fold,seed,phase)
s = read(root/'complete.json')
if (s['status'],s['fold'],s['seed'],s['phase'],s['contract_sha256']) != (
'COMPLETE',fold,seed,phase,sha(ROOT/'contract.json')):
raise ValueError('Stage terminal differs')
return read(checked(root/'worker-records/result.json',s['result_sha256']))
def files_for(result):
entries=result['files']
for row in entries:
checked(row['path'],row['sha256'])
return entries
def validate_result(c, fold, seed, phase):
root=path(fold,seed,phase)
result=root/'worker-records/result.json'
obj=read(result)
ticket=read(root/'started.json')
spawn=read(root/'actual-worker-spawn.json')
uuid=DEVICES[int(ticket['ordinal'])]
if (spawn['physical_uuid']!=uuid or spawn['ordinal']!=ticket['ordinal']
or read(root/'worker-device-verified.json') != dict(
process=spawn['process'],ordinal=ticket['ordinal'],physical_uuid=uuid,observed_uuids=[uuid])
or (obj['status'],obj['fold'],obj['seed'],obj['phase'],obj['test_targets_read']) !=
('COMPLETE',fold,seed,phase,False)):
raise ValueError('Worker, physical card or target scope differs')
files_for(obj)
model=c['folds'][str(fold)]
if phase=='predict':
if [r['case'] for r in obj['files']] != model['parts']['test']:
raise ValueError('Test prediction roster differs')
else:
if (obj['epochs'],obj['learning_cases'],obj['learning_nodes']) != (
1 if phase=='first' else c['epochs'],model['counts']['learning'],model['learning_nodes']):
raise ValueError('Full fold epoch coverage required')
return sha(result)
def run_stage(c, auth, fold, seed, phase, deadline):
if time.time()>=auth['dispatch_before_unix']:
raise RuntimeError('APPROVED_DISPATCH_WINDOW_EXPIRED')
ordinal=next((i for i,u in enumerate(DEVICES) if snapshot(u)['idle']),None)
if ordinal is None:
raise RuntimeError('NO_IDLE_APPROVED_CARD')
started=time.monotonic()
if deadline-started<=c['limits']['cleanup_seconds']:
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
root=path(fold,seed,phase)
root.mkdir(parents=True,exist_ok=False)
ticket=dict(fold=fold,seed=seed,phase=phase,ordinal=str(ordinal),contract_sha256=sha(ROOT/'contract.json'),
booked_gpu_seconds=deadline-started,guard_started_monotonic=started,
dispatch_before_unix=auth['dispatch_before_unix'])
save(root/'started.json',ticket)
command=[c['environment']['python'],'-B',str(SOURCE/'track_b_gino_greif_worker.py'),str(fold),str(seed),phase]
receipt=guard(root,command,uuid=DEVICES[ordinal],limits=dict(c['limits'],cpu_seconds=8*(deadline-started)),
deadline=deadline,ticket=ticket,validate_result=lambda:validate_result(c,fold,seed,phase))
if receipt['status']!='COMPLETE':
raise RuntimeError(f"{fold}/{seed}/{phase} {receipt['trigger']}")
return stage_terminal(fold,seed,phase), receipt['charged_gpu_seconds']
def cost_gate(c, receipts, phase_booked, elapsed):
candidate=dict(folds=c['folds'],epochs=c['epochs'],projection=dict(
accounted_gpu_seconds=phase_booked,upper_remaining_before_other_bookings_gpu_seconds=prep.CAP-phase_booked))
stages=sum(r['booked_gpu_seconds'] for r in receipts.values())
overhead=4*PRED_SECONDS+FIT_SETUP_RESERVE+max(0.,elapsed-stages)
return prep.first_epoch_gate(candidate,receipts,other_unsettled=0.,approved_greif_cap=CAP,remaining_overhead=overhead)
def run(c, auth, phase_booked):
with (ROOT/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (ROOT/'started.json').exists() or any(ROOT.glob('fold-*')):
raise FileExistsError('The combined attempt already exists')
began=time.monotonic()
save(ROOT/'started.json',dict(started_monotonic=began,unix=time.time(),phase_booked=phase_booked,
greif_reserved_gpu_seconds=CAP,stage_authorization_sha256=sha(ROOT/'stage-authorization.json')))
deadline=began+CAP
fit_deadline=deadline-4*PRED_SECONDS
status,trigger='STOPPED',None
def stop(number,_frame):
raise KeyboardInterrupt(signal.Signals(number).name)
handler=signal.signal(signal.SIGTERM,stop)
try:
first={}
for fold in (0,1):
result,charge=run_stage(c,auth,fold,0,'first',min(fit_deadline,time.monotonic()+FIRST_STAGE_MAX))
first[str(fold)]=dict(fold=fold,seed=0,epoch=1,learning_cases=result['learning_cases'],
learning_nodes=result['learning_nodes'],heldout_target_reads=0,
epoch_seconds=result['first_epoch_seconds'],booked_gpu_seconds=charge)
gate=cost_gate(c,first,phase_booked,time.monotonic()-began)
save(ROOT/'first-epoch-cost-gate.json',dict(gate,measurements=first))
if not gate['release_epoch2']:
raise RuntimeError(gate['status'])
models=[]
for fold,seed in JOBS:
result,_=run_stage(c,auth,fold,seed,'fit',fit_deadline)
models.extend(files_for(result))
save(ROOT/'models-frozen.json',models)
predictions=[]
for fold,seed in JOBS:
result,_=run_stage(c,auth,fold,seed,'predict',min(deadline,time.monotonic()+PRED_SECONDS))
predictions.extend(files_for(result))
save(ROOT/'predictions-frozen.json',predictions)
status='PREDICTIONS_FROZEN_NO_TARGETS'
except (Exception,KeyboardInterrupt) as exc:
trigger=str(exc) or type(exc).__name__
finally:
signal.signal(signal.SIGTERM,handler)
charged=time.monotonic()-began
if charged>CAP and status!='STOPPED':
status,trigger='STOPPED','TOTAL_GPU_CAP'
save(ROOT/'complete.json',dict(status=status,trigger=trigger,charged_gpu_seconds=charged,
contract_sha256=sha(ROOT/'contract.json'),test_targets_read=False,no_retry=True))
return 0 if status=='PREDICTIONS_FROZEN_NO_TARGETS' else 1
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('action',choices=('run','pack','score'))
args=parser.parse_args()
c=contract()
auth,booked=authorize(c)
if args.action=='run':
return run(c,auth,booked)
from track_b_gino_greif_custody import pack, score
return pack(c) if args.action=='pack' else score(c)
if __name__=='__main__':
raise SystemExit(main())
+228
View File
@@ -0,0 +1,228 @@
"""One guarded Greif job. Only learning tensors are opened by the fit paths."""
import argparse
import ctypes
import json
import os
from pathlib import Path
import random
import signal
import sys
import time
from types import FunctionType, SimpleNamespace
import numpy as np
import torch
from phase2_h2_data import cached, standardization
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
from phase2_residual_math import decode_seconds, residual_target
from s2_custody_common import prediction_inputs
import track_b_gino_formal_worker as release
import track_b_gino_greif_entry as entry
from track_b_gino_greif_adapter import GreifGINO, stencil
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import cuda_peaks, test_environment
def fold_view(c, fold):
from track_b_gino_greif_prepare import PILOT, PINS
pilot=read(checked(PILOT,PINS[str(PILOT)]))
row=pilot['folds'][fold]
fixed=c['folds'][str(fold)]
if any(row[role]!=cases for role,cases in fixed['parts'].items()) or row['clock_sha256']!=fixed['clock_sha256']:
raise ValueError('Learning, validation and test roles or original clock differ')
return dict(pilot,folds=[row])
def model_input(pilot, case, stats, role):
f=pilot['folds'][0]
if role not in ('learning','test') or case not in f[role]:
raise PermissionError('Original fold role required before reading inputs')
pin=pilot['geometry'][case]
with np.load(checked(pin['path'],pin['sha256']),allow_pickle=False) as archive:
points,raw,mass=(archive[k].copy() for k in ('points','raw_ids','mass'))
if len(raw)!=pilot['parts'][case]['nodes'] or not np.array_equal(raw,cached(pilot,case,'raw_ids')) or not np.array_equal(mass,cached(pilot,case,'mass')):
raise ValueError('Original full node identity differs')
feat=np.asarray(cached(pilot,case,'features'),np.float64)
x=((feat-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
fvec=cached(pilot,case,'Vf')
if x.shape!=(len(raw),19) or not np.isfinite(x).all() or fvec.dtype!=np.float64 or fvec.shape!=(len(raw),):
raise ValueError('Original nineteen columns or FP64 clock differ')
corners,weights=stencil(points,raw)
args=(torch.from_numpy(x),torch.from_numpy(mass.astype(np.float32)),corners,weights)
return args,raw,fvec
def checkpoint(path, model, opt, epoch, fold, seed, stats):
temp=path.with_suffix('.tmp')
with temp.open('xb') as stream:
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),epochs=epoch,fold=fold,seed=seed,
stats=stats,torch_cpu_rng=torch.get_rng_state(),torch_cuda_rng=torch.cuda.get_rng_state(),
numpy_rng=np.random.get_state(),python_rng=random.getstate(),
contract_sha256=sha(entry.ROOT/'contract.json')),stream)
temp.rename(path)
def restore(artifact, fold, seed, epochs):
state=torch.load(checked(artifact['path'],artifact['sha256']),map_location='cpu')
if (state['fold'],state['seed'],state['epochs'],state['contract_sha256']) != (
fold,seed,epochs,sha(entry.ROOT/'contract.json')):
raise ValueError('Checkpoint identity differs')
return state
def train(c, fold, seed, phase, root, records):
p=fold_view(c,fold)
f=p['folds'][0]
rng_seed=seed
random.seed(rng_seed)
np.random.seed(rng_seed)
torch.manual_seed(rng_seed)
torch.cuda.manual_seed_all(rng_seed)
model=GreifGINO().cuda()
opt=optimizer(model)
start, end=1,c['epochs']
if phase=='first':
if seed!=0:
raise PermissionError('First full epoch is seed 0 only')
end=1
normalization=root/'norm'
normalization.mkdir(exist_ok=False)
stats=standardization(p,normalization)
else:
cost=read(entry.ROOT/'first-epoch-cost-gate.json')
if not cost['release_epoch2']:
raise PermissionError('Both measured full epochs must pass the cost gate')
earlier=entry.stage_terminal(fold,0,'first')['files'][0]
state=restore(earlier,fold,0,1)
stats=state['stats']
if seed==0:
start=2
model.load_state_dict(state['model'],strict=True)
opt.load_state_dict(state['optimizer'])
torch.set_rng_state(state['torch_cpu_rng'])
torch.cuda.set_rng_state(state['torch_cuda_rng'])
np.random.set_state(state['numpy_rng'])
random.setstate(state['python_rng'])
del state
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
raise ValueError('Only the learning normalization of this fold is valid')
ncases,nnodes=len(f['learning']),f['learning_nodes']
duration=None
with (root/'learning-access.jsonl').open('x') as access:
for epoch in range(start,end+1):
started=time.monotonic()
model.train()
for group in opt.param_groups:
group['lr']=lr_epoch(epoch)
seen=0
order=np.random.default_rng(np.random.SeedSequence([seed,epoch])).permutation(ncases)
for index in order:
case=f['learning'][int(index)]
args,raw,fvec=model_input(p,case,stats,'learning')
access.write(json.dumps(dict(case=case,fold=fold,seed=seed,epoch=epoch,role='learning',
sha256=p['parts'][case]['output_sha256']['target.npy']))+'\n')
access.flush()
target=residual_target(cached(p,case,'target'),fvec,f['c'],fit_dtype='float32')
opt.zero_grad(set_to_none=True)
prediction=model(*(a.cuda() for a in args))
loss=weighted_loss(prediction,torch.from_numpy(target).cuda(),ncases,len(raw),nnodes)
if not bool(torch.isfinite(loss)):
raise ValueError('Nonfinite full-node H2-weighted loss')
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
opt.step()
if any(not bool(torch.isfinite(value).all()) for value in model.parameters()):
raise ValueError('Nonfinite model weight after update')
torch.cuda.synchronize()
seen+=len(raw)
del prediction,loss,args,raw,fvec,target
if seen!=nnodes:
raise ValueError('Incomplete full-node learning epoch')
duration=time.monotonic()-started
save(records/'progress.json',dict(stage='EPOCH_COMPLETE',epoch=epoch,
learning_cases=ncases,learning_nodes=seen,seconds=duration,**cuda_peaks(torch)))
output=root/f'epoch-{end}.pt'
checkpoint(output,model,opt,end,fold,seed,stats)
return dict(epochs=end,learning_cases=ncases,learning_nodes=nnodes,
first_epoch_seconds=duration if phase=='first' else None,files=[
dict(path=str(output),sha256=sha(output),fold=fold,seed=seed)])
def predict(c, fold, seed, root, records):
from track_b_gino_greif_prepare import QUERY, PINS
p=fold_view(c,fold)
f=p['folds'][0]
models=read(entry.ROOT/'models-frozen.json')
by={(r['fold'],r['seed']):r for r in models}
if len(models)!=4 or set(by)!=set(entry.JOBS):
raise PermissionError('Predict only after four frozen endpoints')
state=restore(by[fold,seed],fold,seed,c['epochs'])
stats=state['stats']
model=GreifGINO().cuda()
model.load_state_dict(state['model'],strict=True)
model.eval()
del state
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
raise ValueError('Original fold normalization differs')
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
entries=[]
with torch.no_grad():
for case in f['test']:
args,raw,fvec=model_input(p,case,stats,'test')
q=prediction_inputs(query[case])
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
raise ValueError('Frozen PPS query-to-node map differs')
residual=model(*(a.cuda() for a in args)).cpu().numpy()
seconds=decode_seconds(residual,fvec,f['c'],p['parts'][case]['t_set'])
item=root/(case+'.npz')
with item.open('xb') as file:
np.savez_compressed(file,seconds=seconds,raw_ids=raw,query_raw_ids=q['raw_ids'],
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg'])
entries.append(dict(case=case,fold=fold,seed=seed,family=query[case]['component_id'],
path=str(item),sha256=sha(item)))
save(records/'progress.json',dict(stage='PREDICT_NO_TARGETS',cases=len(entries),**cuda_peaks(torch)))
return dict(files=entries)
def run(fold,seed,phase):
c=entry.contract()
auth,_=entry.authorize(c)
root=entry.path(fold,seed,phase)
ticket=read(root/'dispatch.json')
parent=os.getppid()
if (ticket['coordinator']!=process_identity(parent) or (ticket['fold'],ticket['seed'],ticket['phase'])!=(fold,seed,phase)
or ticket['contract_sha256']!=sha(entry.ROOT/'contract.json')
or time.monotonic()>=ticket['start_by_monotonic'] or time.time()>=auth['dispatch_before_unix']
or str(Path(sys.executable).absolute())!=c['environment']['python']):
raise PermissionError('Worker not attached to the reviewed guard')
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0) or os.getppid()!=parent:
raise RuntimeError('Could not bind worker lifetime to the guard')
test_environment(c)
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
device=entry.DEVICES[int(ticket['ordinal'])]
binder=FunctionType(release.bind_device.__code__,dict(release.bind_device.__globals__,
formal=SimpleNamespace(DEVICES={str(seed):device})),release.bind_device.__name__)
binder(root,c,ticket,torch)
torch.backends.cuda.matmul.allow_tf32=False
torch.backends.cudnn.allow_tf32=False
torch.cuda.reset_peak_memory_stats()
records=root/'worker-records'
records.mkdir(exist_ok=False)
result=predict(c,fold,seed,root,records) if phase=='predict' else train(c,fold,seed,phase,root,records)
save(records/'result.json',dict(result,status='COMPLETE',fold=fold,seed=seed,phase=phase,
test_targets_read=False,**cuda_peaks(torch)))
if __name__=='__main__':
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('fold',type=int,choices=(0,1))
parser.add_argument('seed',type=int,choices=(0,1))
parser.add_argument('phase',choices=('first','fit','predict'))
args=parser.parse_args()
run(args.fold,args.seed,args.phase)