Count 2012 frozen candidates and start bounded registration with first-100 checkpoint
This commit is contained in:
1 parent
ff439e81fc
commit
d4e26d48d0
17 files changed
+16816
-3
No files matched your search
@@ -0,0 +1,51 @@
|
||||
"""Register frozen candidates and apply the fixed engineering rule, no field labels."""
|
||||
from pathlib import Path
|
||||
import argparse,hashlib,json,os,resource,time
|
||||
import numpy as np
|
||||
from g1_registration_cost import register
|
||||
from g1_candidate_fingerprints import check_contract
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
ROOT=Path(os.environ.get('MELTMIND_G1_REGISTRATION_ROOT','artifacts/s2-g1-registration-2026-11-14'))
|
||||
|
||||
def engineering_rule(routes,rga,rgb,thresholds):
|
||||
ratio=max(rga,rgb)/min(rga,rgb);mean=(rga+rgb)/2;report={};scores=[]
|
||||
for name in ['physical','scale_free']:
|
||||
rows=[r for r in routes if r['route']==name]
|
||||
if len(rows)!=2 or {r['parity'] for r in rows}!={-1,1}:raise ValueError('Missing det branch')
|
||||
branches={str(r['parity']):r['residual_max_two_way_area_pps_p95']/(mean if name=='physical' else 1.) for r in rows}
|
||||
best=min(branches,key=lambda k:(branches[k],k));other='-1' if best=='1' else '1';value=branches[best]
|
||||
if not np.isfinite(value) or value<0 or thresholds[name]<=0:raise ValueError('Invalid residual/threshold')
|
||||
eligible=name=='scale_free' or ratio<=1.05;score=value/thresholds[name]
|
||||
label=('det'+('+' if best=='1' else '-')+' preferred') if eligible and value<thresholds[name] and branches[other]>2*value else 'branch-ambiguous (near-symmetric)'
|
||||
report[name]={'best_residual':value,'size_gate_pass':eligible,'threshold':thresholds[name],'normalized_score':score,
|
||||
'branch_residuals':branches,'branch_label':label}
|
||||
if eligible:scores.append(score)
|
||||
score=min(scores)
|
||||
return {'routes':report,'rg_ratio':ratio,'combined_score':score,
|
||||
'engineering_status':'LINK_UNDER_FIXED_RULE' if score<=1 else 'BAND_REVIEW_REQUIRED' if score<=2 else 'OUTSIDE_FIXED_RULE',
|
||||
'not_a_relatedness_or_recall_certificate':True}
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('index',type=int);args=ap.parse_args();c=check_contract(ROOT)
|
||||
plan=json.loads((ROOT/'custodian/plan.json').read_text());pair=plan['jobs'][args.index];start=time.monotonic();g=[]
|
||||
for case in [pair['a'],pair['b']]:
|
||||
cache=plan['geometry_cache'][case];p=Path(cache['path'])
|
||||
if hashlib.sha256(p.read_bytes()).hexdigest()!=cache['sha256']:raise ValueError('Frozen geometry cache changed')
|
||||
with np.load(p,allow_pickle=False) as z:
|
||||
if set(z.files)!={'points','boundary','center','rg'}:raise ValueError('Unexpected non-geometry cache member')
|
||||
g.append({k:z[k] for k in z.files})
|
||||
loaded=time.monotonic()-start;registration_start=time.monotonic()
|
||||
routes=register(g[0],g[1],int(pair['a'].split('_')[0]),int(pair['b'].split('_')[0]))
|
||||
decision=engineering_rule(routes,float(g[0]['rg']),float(g[1]['rg']),c['thresholds'])
|
||||
dest=ROOT/'custodian/results'/f'pair-{args.index:04d}.json'
|
||||
if dest.exists():raise FileExistsError('Result already exists; no retry')
|
||||
save(dest,{'index':args.index,'pair':pair,'routes':routes,'engineering_rule':decision,
|
||||
'cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),'wall_seconds':time.monotonic()-start,
|
||||
'geometry_load_wall_seconds':loaded,'registration_wall_seconds':time.monotonic()-registration_start,
|
||||
'max_starts_per_branch':max(len(r['starts']) for r in routes),
|
||||
'peak_rss_bytes':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,'simulation_fields_read':False,
|
||||
'geometric_relationship':'UNREVIEWED','components_frozen':False})
|
||||
print(json.dumps({'index':args.index,'status':'REGISTRATION_COMPLETE','engineering_status':decision['engineering_status']}),flush=True)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""First fixed-rank block cost projection; no fitting or changes to k/thresholds."""
|
||||
from statistics import mean
|
||||
|
||||
def first_block_projection(records,total_pairs):
|
||||
if len(records)!=100:raise ValueError('First checkpoint requires exactly 100 terminal pairs')
|
||||
cpu=[r['cpu_seconds'] for r in records];good=[r for r in records if r['status']=='PASS']
|
||||
failed=len(records)-len(good);starts=sum(r.get('max_starts_per_branch',14)>=14 for r in records)
|
||||
central=mean(cpu)*total_pairs/3600
|
||||
successes=mean(r['cpu_seconds'] for r in good)*total_pairs/3600 if good else central
|
||||
loads=[r.get('geometry_load_wall_seconds',0) for r in good]
|
||||
load_delta=max(loads)-mean(loads) if loads else 0
|
||||
sensitivity=max(central,successes)+load_delta*total_pairs/3600
|
||||
by_cohort={str(heavy):[r['cpu_seconds'] for r in good if (r.get('max_starts_per_branch',0)>=14)==heavy] for heavy in [False,True]}
|
||||
reasons=[]
|
||||
if failed/100>.02:reasons.append('failure_rate_over_2_percent')
|
||||
if starts/100>.85:reasons.append('fourteen_or_more_start_share_over_85_percent')
|
||||
if max(central,sensitivity)>100:reasons.append('projected_worker_cpu_over_100_hours')
|
||||
return {'pairs':100,'total_queue_pairs':total_pairs,'mean_worker_cpu_seconds':mean(cpu),
|
||||
'projected_worker_cpu_hours':central,'successful_pair_mean_projection_hours':successes,
|
||||
'high_load_as_cpu_sensitivity_hours':sensitivity,'fourteen_or_more_start_pairs':starts,
|
||||
'fourteen_or_more_start_share':starts/100,'failed_pairs':failed,
|
||||
'worker_cpu_by_start_cohort':{k:mean(v) if v else None for k,v in by_cohort.items()},
|
||||
'geometry_load_wall_seconds_mean':mean(loads) if loads else None,'geometry_load_wall_seconds_max':max(loads) if loads else None,
|
||||
'cache_note':'OS cache not flushed; fingerprint preparation warms it. Measured NPZ/hash load included in worker CPU. High-load sensitivity charges max-minus-mean observed load wall time as extra CPU, not a measured cold-cache guarantee.',
|
||||
'sampling_note':'First 100 in fixed min-rank order; not a random cost sample or certified upper bound',
|
||||
'decision':'STOP_AND_REPORT' if reasons else 'CONTINUE','stop_reasons':reasons}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Freeze the counted <=4000 candidate queue under the already granted G1 budget."""
|
||||
from pathlib import Path
|
||||
import hashlib,json
|
||||
from g1_register_candidates import ROOT
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
|
||||
def main():
|
||||
source=Path('artifacts/s2-g1-candidates-2026-11-14');count=source/'public/candidate-count.json'
|
||||
summary=json.loads(count.read_text());candidates=source/'custodian/candidate-pairs.json'
|
||||
assert summary['status']=='CANDIDATES_COUNTED_NOT_REGISTERED' and summary['union_pairs']<=4000
|
||||
assert sha(candidates)==summary['candidate_sha256']
|
||||
pairs=json.loads(candidates.read_text())['pairs'];assert len(pairs)==summary['union_pairs']
|
||||
assert pairs==sorted(pairs,key=lambda r:(r['min_rank'],r['a'],r['b']))
|
||||
thresholds=Path('artifacts/s2-g1-copy-calibration-2026-11-14/public/calibration-thresholds.json')
|
||||
assert sha(thresholds)==summary['calibration_thresholds_sha256']
|
||||
t=json.loads(thresholds.read_text());assert t['status']=='CALIBRATION_THRESHOLDS_FROZEN'
|
||||
ROOT.mkdir(exist_ok=False);private=ROOT/'custodian';private.mkdir(mode=0o700);(private/'results').mkdir();(ROOT/'public').mkdir()
|
||||
cases=sorted({r[k] for r in pairs for k in ['a','b']});cache={}
|
||||
for case in cases:
|
||||
path=source/'custodian/fingerprints'/case/'geometry.npz';cache[case]={'path':str(path),'sha256':sha(path)}
|
||||
save(private/'plan.json',{'jobs':pairs,'geometry_cache':cache,'candidate_sha256':sha(candidates),'ordering':'frozen min_rank, a, b'})
|
||||
(private/'plan.json').chmod(0o600)
|
||||
code=['prepare_g1_candidate_registration','g1_register_candidates','run_g1_registered_candidates','g1_registration_checkpoint',
|
||||
'g1_registration_cost','g1_candidate_fingerprints','s2_e1_supervisor','audit_samples','boundary_distance']
|
||||
c={'authorizing_rulings':['1794661944.308269','1794662380.787319','1794664166.242149','1794664685.158429','1794664857.650469'],
|
||||
'count_report_before_registration':'1794664837.008399','pairs':len(pairs),'thresholds':t['thresholds'],
|
||||
'code_sha256':{f'tools/{x}.py':sha(f'tools/{x}.py') for x in code},
|
||||
'input_sha256':{str(p):sha(p) for p in [private/'plan.json',count,candidates,thresholds]},
|
||||
'workers':16,'per_pair_wall_seconds':1800,'per_worker_rss_bytes':1342177280,
|
||||
'aggregate_rss_bytes':20*1024**3,'worker_cpu_budget_seconds':100*3600,'wall_seconds':8*3600,'disk_bytes':20*1024**3,
|
||||
'protocol':'Unchanged g1_registration_cost.register; both det branches, both routes, fixed 2000 area PPS, full triangle two-way P95',
|
||||
'engineering_rule':'physical raw/mean Rg with ratio<=1.05, scale_free unchanged; best eligible route/threshold <=1 link; (1,2] band review; >2 outside rule, not certified unrelated',
|
||||
'failure_policy':'retain failed attempts; check >2% at every 100 terminal jobs and final partial batch; no retries/replacements',
|
||||
'first_100_checkpoint':'launch only first 100 fixed-rank pairs until all terminal; report mean CPU and >=14-start share and load-cache sensitivity; stop if projection>100 CPU-h or >=14-start share>85%',
|
||||
'branch_label':'name better det branch only if residual strictly below route tau and other branch >2 times larger; otherwise branch-ambiguous (near-symmetric)',
|
||||
'budget_accounting':'worker and supervision recorded separately; aggregate stop conservatively also includes supervision',
|
||||
'simulation_fields_training_gpu':False,'components_frozen':False,
|
||||
'pending_before_component_freeze':['anonymous threshold-band review; unresolved merges per ruling'],
|
||||
'calibration_mirror_flags_closed_by':'1794664685.158429'}
|
||||
save(ROOT/'contract.json',c)
|
||||
save(ROOT/'public/preparation.json',{'status':'REGISTRATION_QUEUE_FROZEN_NOT_STARTED','pairs':len(pairs),'unique_parts':len(cases),
|
||||
'contract_sha256':sha(ROOT/'contract.json'),'plan_sha256':sha(private/'plan.json'),'candidate_sha256':sha(candidates),
|
||||
'thresholds_sha256':sha(thresholds),'workers':16,'worker_cpu_hours':100,'per_pair_minutes':30,
|
||||
'count_previously_reported':True,'no_new_geometry_extraction':True,'no_new_approval_gate':True})
|
||||
print((ROOT/'public/preparation.json').read_text())
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Frozen min-rank candidate registration; bounded workers, CPU, disk and failure checkpoints."""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from g1_register_candidates import ROOT, check_contract
|
||||
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
|
||||
from g1_registration_checkpoint import first_block_projection
|
||||
|
||||
def guard(index):
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
|
||||
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
|
||||
limit=n*c['per_pair_wall_seconds']
|
||||
r=supervise([sys.executable,'tools/g1_register_candidates.py',str(index)],job,'worker',
|
||||
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
|
||||
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
|
||||
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
def terminate_workers(active):
|
||||
# Guards stay alive to reap killed workers and write failure receipts.
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for child in psutil.Process(p.pid).children():stop_group(child.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
|
||||
def run():
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
|
||||
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
|
||||
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0;first_checkpoint_done=False
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='coordinator_interrupted'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
try:
|
||||
while active or (next_index<len(plan['jobs']) and reason is None):
|
||||
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']) and (first_checkpoint_done or next_index<100):
|
||||
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
|
||||
log=(path/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[i]=(p,log)
|
||||
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
|
||||
for i,(p,log) in list(active.items()):
|
||||
code=p.poll()
|
||||
if code is not None:
|
||||
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
|
||||
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
|
||||
done.append({'index':i,'returncode':code,**result});del active[i]
|
||||
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
|
||||
reason='failure_rate_over_2_percent'
|
||||
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
|
||||
if not first_checkpoint_done and len(done)==100:
|
||||
checkpoint_rows=[]
|
||||
for record in sorted(done,key=lambda r:r['index']):
|
||||
row={'status':record['worker']['status'],'cpu_seconds':record['worker']['cpu_seconds']}
|
||||
path=ROOT/'custodian/results'/f"pair-{record['index']:04d}.json"
|
||||
if path.exists():
|
||||
data=json.loads(path.read_text());row.update(max_starts_per_branch=data['max_starts_per_branch'],geometry_load_wall_seconds=data['geometry_load_wall_seconds'])
|
||||
checkpoint_rows.append(row)
|
||||
checkpoint=first_block_projection(checkpoint_rows,len(plan['jobs']))
|
||||
save(ROOT/'public/first-100-checkpoint.json',checkpoint);first_checkpoint_done=True
|
||||
if checkpoint['decision']=='STOP_AND_REPORT':reason='first_100_projection_stop'
|
||||
print(json.dumps({'event':'FIRST_100_CHECKPOINT',**checkpoint}),flush=True)
|
||||
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
|
||||
if done and len(done)==len(plan['jobs']):
|
||||
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
|
||||
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
|
||||
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
|
||||
live_cpu=live_rss=0
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
|
||||
try:
|
||||
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
except psutil.NoSuchProcess:pass
|
||||
peak_rss=max(peak_rss,live_rss)
|
||||
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
|
||||
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
|
||||
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
|
||||
if reason:terminate_workers(active)
|
||||
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
|
||||
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
|
||||
if active:time.sleep(.5)
|
||||
finally:
|
||||
terminate_workers(active)
|
||||
for p,log in active.values():
|
||||
try:p.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
|
||||
log.close()
|
||||
failures=sum(r['worker']['status']!='PASS' for r in done)
|
||||
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
|
||||
summary={'status':'ALL_REGISTRATIONS_COMPLETE' if passed else 'REGISTRATION_INCOMPLETE',
|
||||
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
|
||||
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
|
||||
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_registration':True,'candidate_generation_completed':True,'no_retries':True}
|
||||
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
|
||||
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
|
||||
return 0 if passed else 1
|
||||
|
||||
if __name__=='__main__':
|
||||
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
|
||||
sys.exit(guard(a.guard) if a.guard is not None else run())
|
||||
@@ -0,0 +1,34 @@
|
||||
import unittest
|
||||
from g1_register_candidates import engineering_rule
|
||||
from g1_registration_checkpoint import first_block_projection
|
||||
|
||||
def routes(p1,p2,s1,s2):
|
||||
return [{'route':route,'parity':parity,'residual_max_two_way_area_pps_p95':value}
|
||||
for route,parity,value in [('physical',1,p1),('physical',-1,p2),('scale_free',1,s1),('scale_free',-1,s2)]]
|
||||
|
||||
class RuleChecks(unittest.TestCase):
|
||||
def test_physical_mean_rg_and_best_branch(self):
|
||||
r=engineering_rule(routes(1,.1,1,2),10,10,{'physical':.02,'scale_free':.02})
|
||||
self.assertEqual(r['engineering_status'],'LINK_UNDER_FIXED_RULE');self.assertAlmostEqual(r['combined_score'],.5)
|
||||
def test_scale_gate_excludes_even_perfect_physical_fit(self):
|
||||
r=engineering_rule(routes(0,0,.03,.04),1,2,{'physical':.02,'scale_free':.02})
|
||||
self.assertFalse(r['routes']['physical']['size_gate_pass']);self.assertEqual(r['engineering_status'],'BAND_REVIEW_REQUIRED')
|
||||
def test_threshold_and_band_endpoints(self):
|
||||
for x,status in [(1,'LINK_UNDER_FIXED_RULE'),(2,'BAND_REVIEW_REQUIRED'),(2.01,'OUTSIDE_FIXED_RULE')]:
|
||||
r=engineering_rule(routes(x,x,x,x),1,1,{'physical':1,'scale_free':1});self.assertEqual(r['engineering_status'],status)
|
||||
def test_branch_label_requires_strict_threshold_and_twofold_gap(self):
|
||||
r=engineering_rule(routes(.1,.15,.1,.15),1,1,{'physical':1,'scale_free':1})
|
||||
self.assertEqual(r['routes']['physical']['branch_label'],'branch-ambiguous (near-symmetric)')
|
||||
r=engineering_rule(routes(.1,.21,.1,.21),1,1,{'physical':1,'scale_free':1})
|
||||
self.assertEqual(r['routes']['physical']['branch_label'],'det+ preferred')
|
||||
def test_first_100_cost_and_start_stop_boundaries(self):
|
||||
rows=[{'status':'PASS','cpu_seconds':90,'max_starts_per_branch':4,'geometry_load_wall_seconds':.2} for _ in range(100)]
|
||||
self.assertEqual(first_block_projection(rows,2012)['decision'],'CONTINUE')
|
||||
for r in rows[:85]:r['max_starts_per_branch']=14
|
||||
self.assertEqual(first_block_projection(rows,2012)['decision'],'CONTINUE')
|
||||
rows[85]['max_starts_per_branch']=14
|
||||
self.assertIn('fourteen_or_more_start_share_over_85_percent',first_block_projection(rows,2012)['stop_reasons'])
|
||||
for r in rows:r['cpu_seconds']=200
|
||||
self.assertIn('projected_worker_cpu_over_100_hours',first_block_projection(rows,2012)['stop_reasons'])
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user