Preserve seven raw-domain failures and freeze explicit cache-only tet continuation

This commit is contained in:
Frank14f committed 2026-11-15 17:41:03 +08:00
1 parent 5c5a9dd525
commit 717d3fae7f
84 files changed
+33870

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
"""Pins and provenance join for the explicitly authorized seven cached continuations."""
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
def contract(root):
root=Path(root);c=read_json(root/'contract.json')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
frozen=read_json(root/'code-freeze.json')
if frozen['contract_sha256']!=sha(root/'contract.json') or not frozen.get('git_commit'):
raise ValueError('Continuation code and contract must be fixed')
if c['continuation_indices']!=[26,194,367,406,470,480,483] or c['scope_ruling']!='1794735162.417219':
raise ValueError('Exactly the seven explicitly approved cached parts')
return c
def prepared(root,c):
canonical=read_json(root/'canonical-preparation.json')
if canonical['status']!='CANONICAL617_TET_DOMAIN_READY' or canonical['contract_sha256']!=sha(root/'contract.json'):
raise ValueError('Canonical input join not ready')
checked(c['original_root']+'/batch-complete.json',canonical['original_batch_sha256'])
checked(root/'batch-complete.json',canonical['continuation_batch_sha256'])
checked(c['domain_audit'],canonical['domain_audit_sha256'])
result={}
for item in canonical['parts']:
p=checked(item['result_path'],item['result_sha256']);r=read_json(p)
for name,h in r['output_sha256'].items():checked(p.parent/name,h)
if r['full_tet_T_end']<=0:raise ValueError('Invalid complete-domain endpoint')
if not item['domain_continuation'] and r['full_tet_T_end']!=r['full_VTU_T_end']:
raise ValueError('Do not reuse a curve normalized to a different endpoint')
result[r['case']]=(r,p.parent)
if len(canonical['parts'])!=617 or set(result)!=set(c['cases']):raise ValueError('Canonical roster changed')
return result
+43
View File
@@ -0,0 +1,43 @@
"""Freeze the explicitly approved seven-part continuation and unified tet endpoint."""
import copy
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s2_recovery_supervisor import save
OLD=Path('artifacts/naive-ordering-2026-11-15')
ROOT=Path('artifacts/naive-domain-continuation-2026-11-15')
def main():
if ROOT.exists():raise FileExistsError('Do not overwrite continuation')
c=copy.deepcopy(read_json(OLD/'contract.json'))
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
batch=read_json(OLD/'batch-complete.json');audit=read_json(OLD/'full-domain-audit.json')
indices=[26,194,367,406,470,480,483]
if batch['passed']!=610 or batch['failed']!=7 or sorted(r['index'] for r in audit['failed'])!=indices:
raise ValueError('Scope differs from explicit final seven-part ruling')
if audit['retained_tet_invalid_nodes'] or audit['raw_vs_tet_end_differences'] or audit['finite_max_vs_tet_end_differences']:
raise ValueError('Uniform endpoint reuse not established by completed audit')
c.update(version='naive-tet-domain-continuation-v1',original_root=str(OLD),
original_contract_sha256=sha(OLD/'contract.json'),original_batch_sha256=sha(OLD/'batch-complete.json'),
domain_ruling='1794734938.283669',mathematical_read='1794735016.559279',scope_ruling='1794735162.417219',
latest_ruling='1794735162.417219',continuation_indices=indices,
domain_audit=str(OLD/'full-domain-audit.json'),
continuation_inputs={str(r['index']):r for r in audit['failed']},
supplemental_audit_CPU_s=audit['CPU_s']+read_json(OLD/'failure-domain-audit-inflight.json')['CPU_s'],
endpoint='last labelled arrival on frozen full retained tet domain, d=0 included; not machine end/V-P/log100percent',
time_scale='Per-fold nested learning-pool component/part-equal mean(full_tet_T_end/t_set)',
new_full_decodes_authorized=0,successful_original_parts_rerun=0,budget_reset=False)
c['code_sha256'].update({str(p):sha(p) for p in Path('tools').glob('*naive_domain*.py')})
for p in [OLD/'contract.json',OLD/'batch-complete.json',OLD/'full-domain-audit.json',OLD/'failure-domain-audit-inflight.json',Path('docs/naive-domain-continuation-contract-2026-11-15.md')]:
c['source_sha256'][str(p)]=sha(p)
ROOT.mkdir();save(ROOT/'contract.json',c)
save(ROOT/'prelaunch.json',dict(status='EXPLICIT_SEVEN_DOMAIN_CONTINUATION_FIXED_NOT_STARTED',indices=indices,
contract_sha256=sha(ROOT/'contract.json'),new_decodes=0,original_successes_reused=610,
prior_worker_CPU_s=batch['worker_CPU_s'],supplemental_audit_CPU_s=c['supplemental_audit_CPU_s']))
print(json.dumps(read_json(ROOT/'prelaunch.json')))
if __name__=='__main__':main()
+255
View File
@@ -0,0 +1,255 @@
"""Frozen clocks, mapped predictions and diagnostic scoring; no model fitting."""
import argparse
from collections import defaultdict
import json
import os
from pathlib import Path
import resource
import sys
import threading
import time
import numpy as np
import psutil
from freeze_g1_components import sha
from s2_custody_common import checked, read_json, prediction_inputs, expected_models, prediction_path
from s2_custody_statistics import part_metrics, component_values, contrast
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from naive_domain_common import contract, prepared
from naive_ordering_common import rows
from naive_ordering_math import pps_cdf, training_clock
ROUTES = {'R2': 'R2', 'ORACLE_FULL_TRAIN': 'R2', 'ORACLE_FULL_END': 'R2',
'ORACLE_MASK_TRAIN': 'R2', 'B0_DW1_CLOCK': 'R2', 'B0_DW2_CLOCK': 'R2',
'REORDER_R2': 'R2', 'REORDER_N': 'N', 'REORDER_E': 'E', 'REORDER_B0': 'R2'}
def validate_manifest(c,index):
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
expected=set()
for row in rows(c):
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
for pool in (25,50,100):
for arm in ('ORACLE_FULL_TRAIN','ORACLE_FULL_END','ORACLE_MASK_TRAIN','B0_DW1_CLOCK','B0_DW2_CLOCK'):
expected.add((row['case'],fold,pool,arm,None))
for model in expected_models(original,fold):
if model['arm']=='B0' or model['pool']==pool:
expected.add((row['case'],fold,pool,'REORDER_'+model['arm'],model['seed']))
actual=[(v['case'],v['fold'],v['pool'],v['arm'],v['seed']) for v in index.values()]
if len(actual)!=len(set(actual)) or set(actual)!=expected:
raise ValueError('Missing/extra/duplicate mapped predictions against frozen original model roster')
def make_maps(root):
c=contract(root);parts=rows(c);prep=prepared(root,c)
if (root/'mapping-started.json').exists():raise FileExistsError('No repeat map phase')
save(root/'mapping-started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
start=time.process_time()
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
original_index=read_json(checked(c['prediction_index'],c['prediction_index_sha256']))
ends={case:r['full_tet_T_end'] for case,(r,_) in prep.items()}
clocks=[]
for fold in range(5):
for pool in (25,50,100):
value,access=training_clock(parts,fold,pool,ends)
clocks.append(dict(fold=fold,pool=pool,c=value,learning_access=access,
definition='mean across components of within-component mean(full_tet_T_end/t_set)',
validation_or_test_used_for_scale=False))
save(root/'training-clocks.json',clocks)
clock={(r['fold'],r['pool']):r['c'] for r in clocks}
save(root/'clock-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
clocks_sha256=sha(root/'training-clocks.json'),batch_sha256=sha(root/'batch-complete.json'),canonical_sha256=sha(root/'canonical-preparation.json'),
before_new_test_mapping=True))
index={};counter=0;curves=[]
for row in parts:
case=row['case'];fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
meta,directory=prep[case];d=prediction_inputs(row)
with np.load(directory/'full-domain-maps.npz',allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'][d['ids']],d['raw_ids']):raise ValueError('Full/query map order')
full={key:z[key][d['ids']].copy() for key in z.files if key!='raw_ids'}
with np.load(directory/'full-node-labels.npz',allow_pickle=False) as z:
truth=z['seconds'][d['raw_ids']].copy()
masked_left,masked_right=pps_cdf(truth,d['counts'])
def output(route,pool,seed,left,right,seconds,source=None):
nonlocal counter
tag=route+('' if seed is None else f'-seed{seed}')
path=root/'predictions'/str(fold)/str(pool)/tag/(case+'.npz')
path.parent.mkdir(parents=True,exist_ok=True)
if path.exists():raise FileExistsError('Duplicate mapped prediction')
extra={}
if route.startswith('REORDER_') or route=='ORACLE_MASK_TRAIN':
# These are plug-in estimates, not certified whole-domain bounds.
alpha=meta['estimated_alpha']
extra=dict(estimated_domain_lower=alpha*left,estimated_domain_upper=alpha*right+1-alpha)
np.savez_compressed(path,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'],F_left=left,F_right=right,**extra)
index[str(path)]=dict(sha256=sha(path),case=case,component_id=row['component_id'],fold=fold,pool=pool,
arm=route,seed=seed,source_prediction=source,oracle=route.startswith('ORACLE_'),
domain='conditional_PPS' if extra else 'full_tet_lumped_volume')
counter+=1
for pool in (25,50,100):
scale=d['t_set']*clock[fold,pool]
output('ORACLE_FULL_TRAIN',pool,None,full['truth_left'],full['truth_right'],scale*full['truth_left'])
output('ORACLE_FULL_END',pool,None,full['truth_left'],full['truth_right'],meta['full_tet_T_end']*full['truth_left'])
output('ORACLE_MASK_TRAIN',pool,None,masked_left,masked_right,scale*masked_left)
output('B0_DW1_CLOCK',pool,None,full['df1_left'],full['df1_right'],scale*full['df1_left'])
output('B0_DW2_CLOCK',pool,None,full['df2_left'],full['df2_right'],scale*full['df2_left'])
for model in expected_models(original,fold):
if model['arm']!='B0' and model['pool']!=pool:continue
p=prediction_path(Path(c['S2_root']),model,case)
with np.load(checked(p,original_index[str(p)]),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Source prediction domain')
score=z['seconds'].copy()
left,right=pps_cdf(score,d['counts']);mapped=scale*left
old_last=d['raw_ids'][score==score.max()].min();new_last=d['raw_ids'][mapped==mapped.max()].min()
if old_last!=new_last:raise ValueError('Order-preserving map changed final location')
if model['arm']=='B0' and not np.allclose(score,d['t_set']*full['df1_left'],rtol=1e-12,atol=1e-12):
raise ValueError('Unscaled Dw fallback differs from frozen B0')
output('REORDER_'+model['arm'],pool,model['seed'],left,right,mapped,str(p))
curve=read_json(directory/'volume-clock-curve.json')
if 'over_full_end' in curve:
if curve['full_VTU_end']!=meta['full_tet_T_end']:raise ValueError('Healthy original curve requires explicit endpoint equality')
curve['over_tet_end']=curve.pop('over_full_end')
curve['T_end_tet_domain']=curve.pop('full_VTU_end')
curves.append(dict(case=case,component_id=row['component_id'],**curve))
save(root/'mapping-progress.json',dict(completed_parts=len(curves),prediction_files=counter,CPU_s=time.process_time()-start))
validate_manifest(c,index)
save(root/'prediction-index.json',index)
save(root/'volume-clock-curves.json',curves)
distribution={}
for field in ('seconds','over_t_set','over_tet_end'):
values=np.array([r[field] for r in curves])
distribution[field]=dict(median=np.median(values,axis=0).tolist(),p10=np.percentile(values,10,axis=0,method='linear').tolist(),
p90=np.percentile(values,90,axis=0,method='linear').tolist())
save(root/'curve-summary.json',dict(part_count=len(curves),aggregation='part-equal descriptive median/p10/p90; no independent-sample inference',
volume_quantiles=curves[0]['volume_quantiles'],**distribution))
save(root/'predictions-complete.json',dict(status='ALL_NAIVE_MAPS_FROZEN',contract_sha256=sha(root/'contract.json'),
code_freeze=read_json(root/'code-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
clock_freeze_sha256=sha(root/'clock-freeze.json'),prediction_files=counter,CPU_s=time.process_time()-start,
full_decode_count=617,original_predictions_unchanged=True,new_model_fits=0))
return 0
def reports(scores,roster,members):
tables=[];contrasts=[];components=[]
for subset in ('all','not_known_exposed'):
for pool in (25,50,100):
primary={}
for route,alias in ROUTES.items():
data=[dict(r,arm=alias) for r in scores if r['arm']==route]
for metric in ('relative_l2','near_wall','kendall_tau_b','last_location'):
values,excluded,details=component_values(data,roster,alias,pool,metric,subset!='all',members)
worst=max(values,key=values.get) if values and metric=='relative_l2' else None
tables.append(dict(subset=subset,pool=pool,arm=route,metric=metric,
mean=float(np.mean(list(values.values()))) if values else None,components=len(values),
parts=sum(details[k]['parts'] for k in values),nodes=sum(details[k]['nodes'] for k in values),
excluded_components=excluded,worst_component=dict(id=worst,value=values[worst]) if worst else None))
components.extend(dict(subset=subset,pool=pool,arm=route,metric=metric,component_id=k,value=v,**details[k]) for k,v in values.items())
if metric=='relative_l2':primary[route]=(values,details)
ref,details=primary['R2']
for route,(candidate,cd) in primary.items():
if route=='R2':continue
pair=set(ref)&set(candidate);excluded=sorted((set(details)|set(cd))-pair)
result=contrast(ref,candidate)
low,high=result.get('low'),result.get('high')
contrasts.append(dict(subset=subset,pool=pool,reference='R2',candidate=route,
oracle=route.startswith('ORACLE_'),exploratory=True,primary_confirmatory=False,
within_descriptive_10percent_band=low is not None and low>=-.1 and high<=.1,
included_folds=sorted({details[k]['fold'] for k in pair}),excluded_components=excluded,
excluded_parts=sum(details[k]['parts'] for k in excluded),excluded_nodes=sum(details[k]['nodes'] for k in excluded),**result))
return tables,contrasts,components
def score_maps(root):
c=contract(root);roster=read_json(c['roster'])['parts'];by_case={r['case']:r for r in roster}
prepared(root,c)
proof=read_json(root/'predictions-complete.json')
if proof['status']!='ALL_NAIVE_MAPS_FROZEN' or proof['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Mapping not frozen')
clock_proof=read_json(checked(root/'clock-freeze.json',proof['clock_freeze_sha256']))
checked(root/'training-clocks.json',clock_proof['clocks_sha256'])
checked(root/'canonical-preparation.json',clock_proof['canonical_sha256'])
index=read_json(checked(root/'prediction-index.json',proof['prediction_index_sha256']))
if len(index)!=proof['prediction_files']:raise ValueError('Manifest size')
validate_manifest(c,index)
for path,row in index.items():checked(path,row['sha256'])
dest=root/'scores';dest.mkdir(exist_ok=False)
start=time.process_time();save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
originals=read_json(checked(c['original_part_scores'],c['original_part_scores_sha256']))
original_lookup={(r['case'],r['arm'],r['pool'],r['seed']):r for r in originals}
scores=[r for r in originals if r['arm']=='R2']
labels={r['case']:r for r in read_json(checked(c['label_index'],c['label_index_sha256']))}
grouped=defaultdict(list)
for path,item in index.items():grouped[item['case']].append((path,item))
unchanged_last=0
for case,items in sorted(grouped.items()):
d=prediction_inputs(by_case[case]);entry=labels[case]
with np.load(checked(entry['path'],entry['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Scoring cache domain')
truth=z['seconds'].copy()
for path,item in items:
with np.load(path,allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Mapped domain')
metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],case.split('_')[0])
if item['arm'].startswith('REORDER_'):
source=item['arm'].removeprefix('REORDER_');pool=None if source=='B0' else item['pool']
old=original_lookup[case,source,pool,item['seed']]
if metrics['last_location']!=old['last_location'] or metrics['kendall_tau_b']!=old['kendall_tau_b']:
raise ValueError('Reorder altered invariant last location or sampled tau')
unchanged_last+=1
scores.append({k:item[k] for k in ('case','component_id','fold','pool','arm','seed')}|metrics)
members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
tables,contrasts,components=reports(scores,roster,members)
save(dest/'part-scores.json',scores);save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components)
save(dest/'result.json',dict(status='NAIVE_ORDERING_COMPLETE',headline='Exploration after S2 results were known; not a fresh confirmatory test',
no_model_training=True,historical_full_custody_decodes=617,new_full_custody_decodes=0,original_guard_failures_retained=7,reorder_invariant_rows=unchanged_last,
eikonal_vs_Dijkstra_tested=False,Dw_distance_reused=True,only_new_distance_exponent=2,
CPU_s=time.process_time()-start,prediction_rows=len(index),
limitations=['Full discrete tet lumped-volume measure, not exact continuous cavity integration',
'T_end is the last labelled arrival on the full retained tet domain; raw outside-domain NaNs retained, no nanmax',
'PPS conditional CDF and estimated alpha do not certify unobserved domain',
'Oracle success does not establish usable learned ordering; poor masked oracle is not a standalone refutation of constant flow',
'Reordering preserves final location and ties; no gate-ranking validation',
'All comparisons exploratory; no new material/process generalization or O1 test'],
output_sha256={p.name:sha(p) for p in dest.glob('*.json')}))
return 0
def guarded(root,phase):
c=contract(root);lim=c['limits'];batch=read_json(root/'batch-complete.json')
prepared(root,c)
spent=batch['worker_CPU_s']
if phase=='score':
previous=read_json(root/'map-guard-result.json')
if previous['worker']['status']!='PASS':raise ValueError('Map phase did not PASS')
spent+=previous['worker']['cpu_seconds']
phase_budget=min(5400 if phase=='map' else 7200,lim['worker_cpu_seconds']-spent,
lim['post_cpu_seconds']-(spent-batch['worker_CPU_s']))
if phase_budget<=0:raise ValueError('No postprocess budget')
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)+disk_usage(Path(c['original_root']))>lim['disk_bytes']:reasons.append('GLOBAL_DISK_LIMIT')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc))
if reasons:
for child in psutil.Process().children(recursive=True):
try:
if child.cmdline()==command:stop_group(child.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
os.sched_setaffinity(0,[c['worker_cpus'][0]])
watcher=threading.Thread(target=watch,daemon=True);watcher.start()
try:
w=supervise(command,root/(phase+'-guard'),'worker',dict(total_cpu_seconds=phase_budget,job_cpu_seconds=phase_budget,
job_wall_seconds=10800,tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();watcher.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/(phase+'-guard-result.json'),dict(worker=w,prior_worker_CPU_s=spent,cumulative_worker_CPU_s=spent+w['cpu_seconds'],
total_cap_CPU_s=lim['worker_cpu_seconds'],postphase_cap_CPU_s=lim['post_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['map','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
raise SystemExit((make_maps(a.root) if a.phase=='map' else score_maps(a.root)) if a.worker else guarded(a.root,a.phase))
+76
View File
@@ -0,0 +1,76 @@
"""Exactly seven never-executed graph/CDF steps, sequential, cumulative accounting."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import sys
import time
from freeze_g1_components import sha
from s2_custody_common import checked,read_json
from s2_recovery_supervisor import save,supervise
from s2_recovery_monitoring import disk_usage
from naive_domain_common import contract
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=contract(root);limits=c['limits'];old=Path(c['original_root'])
if (root/'continuation-started.json').exists():raise FileExistsError('No repeat domain continuation')
os.sched_setaffinity(0,[16]);start=time.monotonic();done=[];reason=None
original=read_json(checked(old/'batch-complete.json',c['original_batch_sha256']))
prior=original['worker_CPU_s']+c['supplemental_audit_CPU_s']
save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(),indices=c['continuation_indices'],
contract_sha256=sha(root/'contract.json'),prior_worker_and_audit_CPU_s=prior,source_code=read_json(root/'code-freeze.json')))
for index in c['continuation_indices']:
spent=prior+sum(j['worker']['cpu_seconds'] for j in done)
if spent+limits['part_cpu_seconds']>limits['preparation_cpu_seconds']:
reason='CUMULATIVE_PREPARATION_CAP';break
if disk_usage(root)+disk_usage(old)>=limits['disk_bytes']:
reason='COMBINED_DISK_CAP';break
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
worker=supervise([sys.executable,'tools/run_naive_domain_part.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=limits['part_cpu_seconds']+1,job_cpu_seconds=limits['part_cpu_seconds'],
job_wall_seconds=limits['part_wall_seconds'],tree_rss_bytes=limits['part_rss_bytes'],
disk_bytes=limits['part_disk_bytes'],max_processes=1),poll=.2)
result=dest/'output/result.json'
if worker['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
worker.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
item=dict(index=index,case=c['cases'][index],worker=worker,result_path=str(result) if result.exists() else None,
result_sha256=sha(result) if result.exists() else None,new_decodes=0,original_failure_retained=True)
save(dest/'guard-result.json',item);done.append(item)
save(root/'progress.json',dict(completed=len(done),current_index=index,cumulative_worker_and_audit_CPU_s=prior+sum(j['worker']['cpu_seconds'] for j in done)))
print(json.dumps(dict(event='DOMAIN_CONTINUATION_TERMINAL',index=index,status=worker['status'])),flush=True)
passed=len(done)==7 and all(j['worker']['status']=='PASS' for j in done) and reason is None
save(root/'batch-complete.json',dict(status='DOMAIN_CONTINUATION_TERMINAL',passed=passed,stop_reason=reason,jobs=done,
prior_original_worker_CPU_s=original['worker_CPU_s'],supplemental_audit_CPU_s=c['supplemental_audit_CPU_s'],
continuation_worker_CPU_s=sum(j['worker']['cpu_seconds'] for j in done),
worker_CPU_s=prior+sum(j['worker']['cpu_seconds'] for j in done),
guard_and_coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
original_status='610_PASS_7_ORIGINAL_GUARD_FAILURES',new_decodes=0,successful_original_parts_rerun=0,
contract_sha256=sha(root/'contract.json')))
if passed:
records=[];continuations={j['index']:j for j in done}
for previous in original['jobs']:
i=previous['index'];continued=i in continuations;j=continuations[i] if continued else previous
if j['worker']['status']!='PASS':raise ValueError('Canonical join contains unsuccessful input')
checked(j['result_path'],j['result_sha256'])
records.append(dict(index=i,case=c['cases'][i],domain_continuation=continued,
original_status=previous['worker']['status'],result_path=j['result_path'],result_sha256=j['result_sha256']))
save(root/'canonical-preparation.json',dict(status='CANONICAL617_TET_DOMAIN_READY',parts=sorted(records,key=lambda r:r['index']),
original_counts=dict(PASS=610,failed=7),new_domain_steps=7,total_custody_decodes=617,new_decodes=0,
contract_sha256=sha(root/'contract.json'),original_batch_sha256=c['original_batch_sha256'],
continuation_batch_sha256=sha(root/'batch-complete.json'),domain_audit_sha256=sha(c['domain_audit']),
common_domain='all vertices of frozen retained tetrahedra, including d=0 and all positive lumped mass',
T_end='last labelled arrival time on retained tet domain; not machine end or V/P switch'))
return 0 if passed else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
+95
View File
@@ -0,0 +1,95 @@
"""Approved cache-only domain continuation; no archive access or decoder."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import shutil
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save
from naive_domain_common import contract
from naive_ordering_common import rows
from naive_ordering_math import cdf_interval, squared_wall_distance, volume_clock_curve
def run(root, index):
c=contract(root)
if index not in c['continuation_indices']:raise PermissionError('Only seven explicitly approved unfinished parts')
row=rows(c)[index];prep=row['preparation'];source=Path(prep['preparation_root'])
dest=root/'parts'/f'part-{index:03d}'/'output'
dest.mkdir(exist_ok=False)
start=time.monotonic();cpu=time.process_time()
save(dest/'started.json',dict(pid=os.getpid(),case=row['case'],index=index,unix=time.time(),
contract_sha256=sha(root/'contract.json'),custody_authorization=c['latest_ruling']))
try:
for name in ['geometry.npz','r2.npz','queries.npz']:checked(source/name,prep['output_sha256'][name])
metadata=read_json(checked(source/'result.json',prep['result_sha256']))
if metadata['r2']['columns'][8]!='Dw':raise ValueError('Expected original wall-distance column')
with np.load(source/'geometry.npz',allow_pickle=False) as z:
x=z['points_raw'];edges=z['edges'];mass=z['node_volume_raw'];raw=z['raw_node_ids'];gate=z['gate_location_raw']
with np.load(source/'r2.npz',allow_pickle=False) as z:
dw=z['features'][:,8];df1=z['Df'];vf=z['Vf']
with np.load(source/'queries.npz',allow_pickle=False) as z:
mask=z['eligible_s2'];ids=z['anchor_ids'][mask];query_raw=z['raw_node_ids'][mask];counts=z['multiplicity'][mask]
if not np.array_equal(raw[ids],query_raw):raise ValueError('Query/raw-ID mismatch')
original=c['continuation_inputs'][str(index)]
label=checked(original['label_path'],original['label_sha256'])
with np.load(label,allow_pickle=False) as z:
full=z['seconds'].copy()
if not np.array_equal(z['retained_tet_raw_ids'],raw):raise ValueError('Frozen tet geometry/raw IDs changed')
inside=np.zeros(len(full),bool);inside[raw]=True
invalid=(~np.isfinite(full))|(full<0)
if not np.array_equal(invalid,~inside) or not np.isnan(full[~inside]).all():
raise ValueError('Continuation is limited to observed outside-tet NaN pattern')
if int(invalid.sum())!=original['invalid_outside_tet'] or np.any(full[raw]<0) or not np.isfinite(full[raw]).all() or full[raw].max()<=0:
raise ValueError('Invalid labels inside frozen tet domain; do not filter by finiteness')
shutil.copy2(label,dest/'full-node-labels.npz')
if sha(dest/'full-node-labels.npz')!=original['label_sha256']:raise ValueError('Raw NaN cache bytes changed')
save(dest/'cache-reuse-receipt.json',dict(case=row['case'],original_label_path=str(label),
label_sha256=original['label_sha256'],new_decodes=0,original_decode_count=1,
invalid_outside_tet=int(invalid.sum()),invalid_in_tet=0,domain_ruling=c['domain_ruling'],scope_ruling=c['scope_ruling']))
old={v['case']:v for v in read_json(checked(c['label_index'],c['label_index_sha256']))}[row['case']]
with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],query_raw) or not np.array_equal(z['counts'],counts) or not np.array_equal(z['seconds'],full[query_raw]):
raise ValueError('Full decode differs from immutable S2 query cache')
left1,right1=cdf_interval(df1,mass)
difference=float(np.max(np.abs(left1-vf)))
if difference>1e-12:raise ValueError('Existing B0 is not the expected full-volume strict D_w Dijkstra CDF')
save(dest/'progress.json',dict(stage='Dw_squared_graph',cpu_seconds=time.process_time()-cpu))
df2,gate_node,zero_edges=squared_wall_distance(x,edges,dw,gate)
left2,right2=cdf_interval(df2,mass)
leftT,rightT=cdf_interval(full[raw],mass)
np.savez_compressed(dest/'full-domain-maps.npz',raw_ids=raw,truth_left=leftT,truth_right=rightT,
df1_left=left1,df1_right=right1,df2_left=left2,df2_right=right2)
end=float(full[raw].max());tet_end=end;query_end=float(full[query_raw].max())
positive=float(mass[dw>0].sum()/mass.sum());K=int(counts.sum());alpha=positive*K/metadata['draw_slots']
curve=volume_clock_curve(full[raw],mass,end,prep['t_set'])
curve['T_end_tet_domain']=curve.pop('full_VTU_end')
curve['over_tet_end']=curve.pop('over_full_end')
curve['raw_VTU_end']=None
curve['outside_tet_NaN_retained']=int(invalid.sum())
save(dest/'volume-clock-curve.json',curve)
save(dest/'result.json',dict(status='COMPLETE',case=row['case'],index=index,component_id=row['component_id'],
t_set=prep['t_set'],full_VTU_T_end=None,full_tet_T_end=tet_end,query_T_max=query_end,
raw_nodes=len(full),tet_nodes=len(raw),zero_mass_excluded_VTU_nodes=len(full)-len(raw),
full_vs_tet_end_difference=None,eligible_slots=K,draw_slots=metadata['draw_slots'],
positive_mass_fraction=positive,estimated_alpha=alpha,alpha_is_estimate=True,
DKW_halfwidth_if_fixed_score_independent_PPS=float(np.sqrt(np.log(40)/(2*K))),
baseline_Dw_Vf_max_abs_difference=difference,Dw_solve_reused=True,Dw2_solver='tet-edge Dijkstra approximation',
zero_midpoint_edges_excluded=zero_edges,gate_node=gate_node,
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,
output_sha256={p.name:sha(p) for p in dest.glob('*') if p.is_file() and p.name not in ['progress.json']},
no_training=True,no_GPU=True,decode_count=0,cache_reused=True,outside_tet_NaN_retained=int(invalid.sum())))
return 0
except Exception as exc:
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,retry=False))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
raise SystemExit(run(a.root,a.index))
+138
View File
@@ -0,0 +1,138 @@
"""Synthetic checks for the labelled seven-part domain continuation."""
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from zipfile import ZipFile
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import read_json
from s2_models.features import R2_COLUMNS
from naive_ordering_math import volume_clock_curve
import run_naive_domain_part as part
import run_naive_domain_analysis as analysis
import run_naive_domain_continuation as continuation
import naive_domain_common as common
class DomainChecks(unittest.TestCase):
def test_cached_NaNs_preserved_without_any_archive_decode(self):
self.cached_case(False)
def test_domain_is_geometry_first_and_missing_tet_label_is_not_filtered(self):
self.cached_case(True)
def cached_case(self,missing_inside):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);source=root/'source';source.mkdir();(root/'parts/part-000').mkdir(parents=True)
x=np.array([[0,0,0],[1,0,0],[0,1,0],[0,0,1.]])
edges=np.array([[0,1],[0,2],[0,3],[1,2],[1,3],[2,3]])
np.savez(source/'geometry.npz',points_raw=x,edges=edges,node_volume_raw=np.ones(4),raw_node_ids=np.arange(4),gate_location_raw=x[0])
features=np.zeros((4,20));features[:,8]=[0,1,1,1]
np.savez(source/'r2.npz',features=features,Df=[0,2,2,2],Vf=[0,.25,.25,.25])
np.savez(source/'queries.npz',eligible_s2=[True]*3,anchor_ids=[1,2,3],raw_node_ids=[1,2,3],multiplicity=[1,2,1])
(source/'result.json').write_text(json.dumps(dict(r2=dict(columns=list(R2_COLUMNS)),draw_slots=4)))
full=np.array([0.,.2,.4,.8,np.nan]);full[0]=np.nan if missing_inside else 0.;old=root/'old.npz';np.savez(old,seconds=full[[1,2,3]],raw_ids=[1,2,3],counts=[1,2,1])
labels=root/'labels.json';labels.write_text(json.dumps([dict(case='case',path=str(old),sha256=sha(old))]))
archive=root/'archive.zip'
with ZipFile(archive,'w') as z:z.writestr('case_sim.vtu','<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece NumberOfPoints="5"><PointData><DataArray Name="Fill_time">x</DataArray></PointData></Piece></UnstructuredGrid></VTKFile>')
st=archive.stat();(root/'contract.json').write_text('{}')
c=dict(archive=str(archive),archive_stat=dict(bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino),label_index=str(labels),label_index_sha256=sha(labels),latest_ruling='synthetic')
cache=root/'original-full-cache.npz';np.savez_compressed(cache,seconds=full,retained_tet_raw_ids=np.arange(4))
c.update(continuation_indices=[0],continuation_inputs={'0':dict(label_path=str(cache),label_sha256=sha(cache),invalid_outside_tet=1)},domain_ruling='synthetic',scope_ruling='synthetic',archive='/does-not-exist')
prep=dict(preparation_root=str(source),t_set=1.,output_sha256={n:sha(source/n) for n in ['geometry.npz','r2.npz','queries.npz']},result_sha256=sha(source/'result.json'))
row=dict(case='case',component_id='A',preparation=prep)
with patch.object(part,'contract',return_value=c),patch.object(part,'rows',return_value=[row]):
if missing_inside:
with self.assertRaisesRegex(ValueError,'outside-tet NaN pattern'):part.run(root,0)
return
self.assertEqual(part.run(root,0),0)
with self.assertRaises(FileExistsError):part.run(root,0)
self.assertEqual(sha(root/'parts/part-000/output/full-node-labels.npz'),sha(cache))
r=read_json(root/'parts/part-000/output/result.json')
self.assertIsNone(r['full_VTU_T_end']);self.assertEqual(r['full_tet_T_end'],.8);self.assertEqual(r['query_T_max'],.8)
self.assertEqual(r['baseline_Dw_Vf_max_abs_difference'],0.)
self.assertEqual(r['raw_nodes'],5);self.assertEqual(r['tet_nodes'],4);self.assertEqual(r['decode_count'],0)
curve=read_json(root/'parts/part-000/output/volume-clock-curve.json')
self.assertEqual(curve['T_end_tet_domain'],.8);self.assertNotIn('full_VTU_end',curve)
def test_canonical_mapping_scoring_uses_tet_end_and_preserves_order(self):
from s2_custody_common import prediction_path
from s2_custody_statistics import part_metrics
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);old=root/'old';old.mkdir();case='00000001_case'
(root/'contract.json').write_text('{}');(root/'code-freeze.json').write_text('{}');(root/'batch-complete.json').write_text('{}');(root/'canonical-preparation.json').write_text('{}')
s2=old/'contract.json';s2.write_text('{}')
row=dict(case=case,component_id='A',included=True,nodes=3,exposure=dict(development_exposed=False),
assignments=[dict(fold=f,role='test' if f==0 else 'learning',learning_pools=['25','50','100']) for f in range(5)])
roster=root/'roster.json';roster.write_text(json.dumps(dict(parts=[row])))
truth=np.array([1.,2.,4.]);raw=np.arange(3);counts=np.array([1,2,1]);left=np.array([0.,1/3,2/3]);right=left+1/3
d=dict(ids=raw,raw_ids=raw,counts=counts,t_set=4.,near_wall=np.array([True,False,True]),coordinates_over_Rg=np.array([[0,0,0],[1,0,0],[2,0,0]]))
prepared=root/'part';prepared.mkdir()
np.savez(prepared/'full-domain-maps.npz',raw_ids=raw,truth_left=left,truth_right=right,df1_left=left,df1_right=right,df2_left=left,df2_right=right)
np.savez(prepared/'full-node-labels.npz',seconds=truth)
(prepared/'volume-clock-curve.json').write_text(json.dumps(volume_clock_curve(truth,np.ones(3),4.,4.)))
models=[]
for pool in [25,50,100]:
for arm in ['R2','N','E']:
for seed in ([None] if arm=='R2' else [0,1,2]):
models.append(dict(fold=0,pool=pool,arm=arm,seed=seed,name=f'{arm}-{pool}-{seed}'))
models.append(dict(fold=0,pool=None,arm='B0',seed=None,name='B0'))
index={};scores=[]
for m in models:
pred=4*left if m['arm']=='B0' else np.array([.9,2.2,3.7])
p=prediction_path(old,m,case);p.parent.mkdir(parents=True,exist_ok=True);np.savez(p,seconds=pred,raw_ids=raw,counts=counts)
index[str(p)]=sha(p)
scores.append(dict(case=case,component_id='A',fold=0,pool=m['pool'],arm=m['arm'],seed=m['seed'],
**part_metrics(pred,truth,counts,d['near_wall'],raw,d['coordinates_over_Rg'],1)))
pi=old/'index.json';pi.write_text(json.dumps(index));sp=old/'scores.json';sp.write_text(json.dumps(scores))
lp=old/'labels.npz';np.savez(lp,seconds=truth,raw_ids=raw,counts=counts)
li=old/'labels.json';li.write_text(json.dumps([dict(case=case,path=str(lp),sha256=sha(lp))]))
ex=root/'exposure.json';ex.write_text(json.dumps([dict(parts=[case])]))
c=dict(roster=str(roster),S2_contract=str(s2),S2_contract_sha256=sha(s2),prediction_index=str(pi),prediction_index_sha256=sha(pi),S2_root=str(old),
original_part_scores=str(sp),original_part_scores_sha256=sha(sp),label_index=str(li),label_index_sha256=sha(li),exposure_membership=str(ex),exposure_membership_sha256=sha(ex))
prep={case:(dict(full_tet_T_end=4.,estimated_alpha=.6),prepared)}
with patch.object(analysis,'contract',return_value=c),patch.object(analysis,'prepared',return_value=prep),patch.object(analysis,'rows',return_value=[row]),patch.object(analysis,'prediction_inputs',return_value=d),patch.object(analysis,'expected_models',return_value=models),patch.object(analysis,'training_clock',return_value=(1.1,[])):
self.assertEqual(analysis.make_maps(root),0)
self.assertEqual(analysis.score_maps(root),0)
result=read_json(root/'scores/result.json');self.assertEqual(result['prediction_rows'],39);self.assertEqual(result['reorder_invariant_rows'],24)
mapped=read_json(root/'prediction-index.json');mapped.pop(next(iter(mapped)))
with self.assertRaisesRegex(ValueError,'Missing/extra'):analysis.validate_manifest(c,mapped)
def test_exact_seven_join_reuses610_and_charges_prior_CPU(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp)/'new';root.mkdir();old=Path(temp)/'old';old.mkdir()
indices=[26,194,367,406,470,480,483];jobs=[];cases=[f'case-{i}' for i in range(617)]
for i in range(617):
passed=i not in indices;p=old/f'result-{i}.json'
p.write_text(json.dumps(dict(case=cases[i],full_tet_T_end=1.,full_VTU_T_end=1.,output_sha256={})))
jobs.append(dict(index=i,worker=dict(status='PASS' if passed else 'STOPPED_OR_FAILED'),result_path=str(p) if passed else None,result_sha256=sha(p) if passed else None))
batch=old/'batch-complete.json';batch.write_text(json.dumps(dict(jobs=jobs,worker_CPU_s=641.9)))
original_hash=sha(batch);audit=old/'domain.json';audit.write_text('{}')
(root/'contract.json').write_text('{}');(root/'code-freeze.json').write_text('{}')
c=dict(original_root=str(old),original_batch_sha256=original_hash,domain_audit=str(audit),continuation_indices=indices,
supplemental_audit_CPU_s=2.4,cases=cases,limits=dict(part_cpu_seconds=180,part_wall_seconds=600,part_rss_bytes=4*2**30,
part_disk_bytes=2*2**30,preparation_cpu_seconds=28800,disk_bytes=20*2**30))
attempted=[]
def supervise(command,dest,name,limits,**kwargs):
i=int(command[-1]);attempted.append(i);out=dest/'output';out.mkdir()
(out/'result.json').write_text(json.dumps(dict(status='COMPLETE',case=cases[i],full_tet_T_end=1.,full_VTU_T_end=None,output_sha256={})))
return dict(status='PASS',cpu_seconds=1.,stop_reason=None)
with patch.object(continuation,'contract',return_value=c),patch.object(continuation.os,'sched_setaffinity'),patch.object(continuation,'supervise',supervise),patch('builtins.print'):
self.assertEqual(continuation.run(root),0)
with self.assertRaises(FileExistsError):continuation.run(root)
self.assertEqual(attempted,indices);self.assertEqual(sha(batch),original_hash)
result=read_json(root/'batch-complete.json');self.assertAlmostEqual(result['worker_CPU_s'],651.3)
join=read_json(root/'canonical-preparation.json');self.assertEqual(len(join['parts']),617)
self.assertEqual(sum(p['domain_continuation'] for p in join['parts']),7)
self.assertEqual(join['original_counts'],dict(PASS=610,failed=7))
self.assertEqual(len(common.prepared(root,c)),617)
def test_scope_rejects_any_eighth_part(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);c=dict(source_sha256={},code_sha256={},continuation_indices=[26,194,367,406,470,480,483,500],scope_ruling='1794735162.417219')
(root/'contract.json').write_text(json.dumps(c));(root/'code-freeze.json').write_text(json.dumps(dict(git_commit='synthetic',contract_sha256=sha(root/'contract.json'))))
with self.assertRaises(ValueError):common.contract(root)
if __name__=='__main__':unittest.main(verbosity=2)