Preserve S3-A interruption and freeze authorized 159-job continuation

This commit is contained in:
Frank14f committed 2026-11-15 18:09:20 +08:00
1 parent 087f63b057
commit 259d1d685b
253 files changed
+51609

No files matched your search

+80
View File
@@ -0,0 +1,80 @@
"""Claude1794736668: exact-case descriptive join of already frozen results only."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import sys
import time
from s2_recovery_supervisor import save, supervise
def read_checked(path, digest):
raw=Path(path).read_bytes()
if hashlib.sha256(raw).hexdigest()!=digest:raise ValueError('Source changed')
return json.loads(raw)
def worker(root):
c=json.loads((root/'contract.json').read_text());start=time.process_time()
src={p:read_checked(p,h) for p,h in c['source_sha256'].items()}
logs=next(v for p,v in src.items() if p.endswith('/index.json'))
scores=next(v for p,v in src.items() if p.endswith('/part-scores.json'))
by_case={Path(v['member']).stem:v for v in logs}
if len(by_case)!=624 or len(logs)!=624:raise ValueError('Duplicate/missing LOG keys')
arms=c['arms'];parts={}
for row in scores:
if row['pool']!=100 or row['arm'] not in arms:continue
if row['seed'] is not None:raise ValueError('Oracle should be deterministic')
case=row['case'];d=parts.setdefault(case,dict(case=case,fold=row['fold'],component_id=row['component_id'],oracle_relative_l2={}))
if row['arm'] in d['oracle_relative_l2']:raise ValueError('Duplicate oracle case')
d['oracle_relative_l2'][row['arm']]=row['relative_l2']
if len(parts)!=617:raise ValueError('Oracle part count')
for case,row in parts.items():
if set(row['oracle_relative_l2'])!=set(arms):raise ValueError('Incomplete oracle arms')
log=by_case[case];d=log['diagnostic']
if not d['quantitative_defined']:raise ValueError('Undefined LOG diagnostic')
row.update(log_report=log['report'],log_report_sha256=log['report_sha256'],
log_max_linearity_deviation_pp=d['linearity']['max_deviation_percentage_points'],
log_within_original_1pp=d['linearity']['within_fixed_tolerance'],
log_first_printed_time_s=d['first_row']['time_s'],
log_first_printed_fill_percent=d['first_row']['fill_percent'],
log_printed_100_time_s=d['last_row']['time_s'],
log_explicit_switch_rows=d['observed_switch_rows'],
log_equal_printed_time_intervals=d['equal_printed_time_intervals'])
unmatched=sorted(set(by_case)-set(parts))
if len(unmatched)!=7:raise ValueError('LOG-only count')
data=[parts[k] for k in sorted(parts)]
save(root/'joined-parts.json',data);save(root/'log-only-parts.json',unmatched)
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig,axes=plt.subplots(1,3,figsize=(12,3.7),layout='constrained')
for ax,arm in zip(axes,arms):
ax.scatter([r['log_max_linearity_deviation_pp'] for r in data],
[r['oracle_relative_l2'][arm] for r in data],s=10,alpha=.5)
ax.set(xlabel='LOG maximum deviation (percentage points)',ylabel='Per-part relative L2',title=arm)
fig.suptitle('617 exact-case joins; descriptive only, no fitted relation or significance test')
fig.savefig(root/'oracle-log-join.png',dpi=170);plt.close(fig)
save(root/'result.json',dict(status='EXACT_CASE_JOIN_COMPLETE',parts=len(data),log_only_parts=unmatched,
new_decodes=0,new_fits=0,new_predictions=0,new_significance_tests=0,
explanatory_conclusion='No causal or universal time-map conclusion drawn from the scatter.',
worker_analysis_CPU_s=time.process_time()-start,
output_sha256={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in root.iterdir() if p.name in ('joined-parts.json','log-only-parts.json','oracle-log-join.png')}))
return 0
def guard(root):
c=json.loads((root/'contract.json').read_text());lim=c['limits'];os.sched_setaffinity(0,[24])
r=supervise([sys.executable,__file__,str(root),'--worker'],root,'worker',dict(
total_cpu_seconds=lim['CPU_s'],job_cpu_seconds=lim['CPU_s'],job_wall_seconds=lim['wall_s'],
tree_rss_bytes=lim['RSS_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
previous=json.loads(Path('artifacts/moldflow-time-map-2026-11-15/report-receipt.json').read_text())
save(root/'guard-result.json',dict(worker=r,prior_LOG_task_CPU_s=previous['cumulative_task_CPU_s'],
cumulative_LOG_task_CPU_s=previous['cumulative_task_CPU_s']+r['cpu_seconds'],LOG_task_CPU_cap=3600))
return 0 if r['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args()
raise SystemExit(worker(a.root) if a.worker else guard(a.root))
+245
View File
@@ -0,0 +1,245 @@
"""Explicit 4+155 infrastructure continuation. Original workers and scientific contract unchanged."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
from run_s2_pilot_preparation import terminate
from run_s2_r2 import finalize_selections
from s3a_budget import first_fit_projection, can_dispatch
from s3a_runtime import contract
from s3a_recovery_common import state, canonical_accounting, selection_ready, manifest, ORDER
FIRST = [0, 12]
def dispatch_order():
return list(ORDER)
def guard(root, index, slot):
if index not in ORDER:
raise ValueError('Original successful jobs may not be dispatched again')
c = contract(root)
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
limits = c['limits']
dest = root/'jobs'/f'job-{index:03d}'
dest.mkdir(parents=True, exist_ok=True)
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
path = dest/'output/result.json'
result = read_json(path) if path.exists() else None
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
if w['status'] == 'PASS':
checked(dest/'output/model.joblib', result['model_sha256'])
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
return 0 if w['status'] == 'PASS' else 1
def first_check(c, done):
samples = {}
for i in FIRST:
item = next(r for r in done if r['index'] == i)
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
return first_fit_projection(c, samples)
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
_, scope, old, _ = state(root)
merged = manifest(root, scope, old, done)
save(root/'two-source-manifest.json', merged)
attempts = {r['index']: r for r in done}
finals = []
for job in c['jobs'][360:]:
row = attempts.get(job['index'])
passed = row is not None and row['worker']['status'] == 'PASS'
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
if passed:
checked(model_path, result['model_sha256'])
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
status='PASS' if passed else 'NA',
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
complete = reason is None and len(attempts) == 390 and accounting_complete
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
worker_cpu_seconds=canonical_accounting(old, done),
original_worker_cpu_seconds=old['worker_cpu_seconds'],
original_aborted_jobs_retained=[230,232,233,234],
two_source_manifest_sha256=sha(root/'two-source-manifest.json'),
guard_cpu_seconds=old['guard_cpu_seconds']+sum(r['supervisor_cpu_seconds'] for r in done if r['index'] in ORDER),
original_coordinator_cpu_seconds=old['coordinator_cpu_seconds'],
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root)+disk_usage(Path(scope['original_root'])), wall_seconds=time.monotonic()-started,
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
save(root/'batch-complete.json', result)
if complete:
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
hpo_terminal_indices=list(range(360)), final_fits=finals,
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
test_predictions_started=False))
return 0 if complete else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c, scope, old, original_passed = state(root)
if (root/'batch-started.json').exists():
raise FileExistsError('No restart/retry of an attempted S3-A queue')
os.sched_setaffinity(0, c['coordinator_cpus'])
start = time.monotonic()
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
active, done = {}, list(original_passed)
order = dispatch_order()
cursor = 0
reason = None
accounting_complete = True
first_checked = True
selections = None
live_cpu = live_rss = disk = 0
monitor = Monitor(root/'monitor-errors.jsonl')
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason = 'S3A_RECOVERY_COORDINATOR_INTERRUPTED'
save(root/'signal-received.json', dict(signal_number=_[0], unix=time.time(), pid=os.getpid()))
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
def terminate_active():
terminate({k: (v['process'], v['log']) for k, v in active.items()})
def collect():
nonlocal reason, accounting_complete
for slot, item in list(active.items()):
code = item['process'].poll()
if code is None:
continue
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
reason = 'MISSING_GUARD_RECEIPT'
accounting_complete = False
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
else:
row = read_json(path)
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
raise ValueError('Invalid job/accounting receipt')
if row.get('result_path'):
checked(row['result_path'], row['result_sha256'])
done.append(dict(index=item['index'], **row))
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
status=row['worker']['status'], completed=len(done))), flush=True)
item['log'].close()
del active[slot]
try:
while active or (cursor < len(order) and reason is None):
collect()
def usage():
values = []
for v in active.values():
value = tree_usage(v['process'].pid)
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
values.append((v['cpu_peak'], value[1]))
return sum(v[0] for v in values), sum(v[1] for v in values)
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
disk, ok = monitor.sample('disk', lambda: disk_usage(root)+disk_usage(Path(scope['original_root'])), disk)
fresh &= ok
spent = canonical_accounting(old, done)
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
if over:
reason = over
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
if not first_checked and len(done) == 2 and not active and reason is None:
check = first_check(c, done)
save(root/'first-fit-cost-check.json', check)
first_checked = True
if not check['continue_within_cap']:
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
if selection_ready(done, active) and selections is None and reason is None:
selections = finalize_selections(root, c, done)
if reason:
terminate_active()
ceiling = len(order)-30 if selections is None else len(order)
parallel = 1 if not first_checked else 4
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
i = order[cursor]
job = c['jobs'][i]
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
cursor += 1
continue
# Conservatively reserve full caps of active workers plus the next task.
cap = c['limits']['per_fit_cpu_seconds']
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
if not active:
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
break
slot = next(k for k in range(4) if k not in active)
dest = root/'jobs'/f'job-{i:03d}'
dest.mkdir(parents=True, exist_ok=True)
log = (dest/'guard.log').open('x')
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
cursor += 1
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
if active or not fresh:
time.sleep(.5)
except Exception as exc:
reason = 'COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
terminate_active()
for item in active.values():
item['process'].wait()
collect()
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('root', type=Path)
p.add_argument('--guard', type=int)
p.add_argument('--slot', type=int)
a = p.parse_args()
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))
+87
View File
@@ -0,0 +1,87 @@
"""Explicit two-source S3-A infrastructure continuation; original sources unchanged."""
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s3a_runtime import contract
RECOVER = [230, 232, 233, 234]
NEVER = list(range(235, 390))
ORDER = RECOVER + NEVER
def validate_scope(c, old, scope):
if scope['recovery_indices'] != RECOVER or scope['never_attempted_indices'] != NEVER:
raise ValueError('Only four explicit recoveries and 155 never attempted are authorized')
if old['status'] != 'S3A_STOPPED_INCOMPLETE' or old['stop_reason'] != 'S3A_COORDINATOR_INTERRUPTED' or not old['accounting_complete']:
raise ValueError('Unexpected original terminal')
attempted = {r['index']: r for r in old['jobs']}
if len(old['jobs']) != 235 or set(attempted) != set(range(235)) or old['not_run_indices'] != NEVER:
raise ValueError('Original exact attempt set differs')
passed=[]
for i,row in attempted.items():
if row['job'] != c['jobs'][i]:raise ValueError('Original specification changed')
if i in RECOVER:
if row['worker']['status'] != 'STOPPED_OR_FAILED' or row['worker']['returncode'] != -9:
raise ValueError('Expected aborted original worker')
else:
if row['worker']['status'] != 'PASS':raise ValueError('Cannot expand recovery to another failed job')
passed.append(row)
if scope['original_worker_CPU_s'] != old['worker_cpu_seconds']:
raise ValueError('Prior CPU charge changed')
if abs(sum(r['worker']['cpu_seconds'] for r in old['jobs'])-old['worker_cpu_seconds']) > 1e-6:
raise ValueError('Incomplete original ledger')
return sorted(passed,key=lambda r:r['index'])
def canonical_accounting(old, done):
ids=[r['index'] for r in done]
if len(ids)!=len(set(ids)):raise ValueError('Duplicate canonical index')
original_good={r['index']:r for r in old['jobs'] if r['worker']['status']=='PASS'}
if not set(original_good)<=set(ids):raise ValueError('Dropped original success')
new=[]
for row in done:
i=row['index']
if i in original_good:
if row!=original_good[i]:raise ValueError('Original success modified')
elif i not in ORDER:raise ValueError('Unauthorized new index')
else:new.append(row)
return old['worker_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in new)
def selection_ready(done, active):
return not active and {r['index'] for r in done}==set(range(360))
def state(root, require_code_freeze=True):
root=Path(root);c=contract(root);scope=read_json(root/'recovery-scope.json')
original=Path(scope['original_root'])
checked(original/'contract.json',sha(root/'contract.json'))
old=read_json(checked(original/'batch-complete.json',scope['original_batch_sha256']))
passed=validate_scope(c,old,scope)
for path,digest in scope['original_evidence_sha256'].items():checked(path,digest)
for row in passed:
i=row['index'];oldpath=original/'jobs'/f'job-{i:03d}'/'guard-result.json'
checked(root/'jobs'/f'job-{i:03d}'/'guard-result.json',sha(oldpath))
if require_code_freeze:
f=read_json(root/'recovery-code-freeze.json')
if not f['git_commit'] or f['scope_sha256']!=sha(root/'recovery-scope.json'):
raise ValueError('Recovery must be fixed before execution')
for p,digest in f['source_sha256'].items():checked(p,digest)
return c,scope,old,passed
def manifest(root,scope,old,done):
original=Path(scope['original_root']);oldgood={r['index'] for r in old['jobs'] if r['worker']['status']=='PASS'}
canonical_accounting(old,done)
records=[]
for row in sorted(done,key=lambda r:r['index']):
i=row['index'];origin=original if i in oldgood else Path(root)
guard=origin/'jobs'/f'job-{i:03d}'/'guard-result.json'
# A final fit with all HPO trials failed has a logical NA, not a worker attempt.
records.append(dict(index=i,job=row['job'],origin='original_success' if i in oldgood else
('explicit_infrastructure_recovery' if i in RECOVER else 'never_attempted_continuation'),
guard_path=str(guard) if guard.exists() else None,guard_sha256=sha(guard) if guard.exists() else None,
result_path=row.get('result_path'),result_sha256=row.get('result_sha256'),status=row['worker']['status']))
return dict(original_batch_sha256=scope['original_batch_sha256'],original_aborted_indices_retained=RECOVER,
original_root=str(original),recovery_root=str(root),records=records,
original_worker_CPU_s=old['worker_cpu_seconds'],cumulative_worker_CPU_s=canonical_accounting(old,done))
+55
View File
@@ -0,0 +1,55 @@
import copy
import unittest
from s3a_recovery_common import RECOVER, NEVER, ORDER, validate_scope, canonical_accounting, selection_ready
def fixture():
c=dict(jobs=[dict(index=i,kind='HPO' if i<360 else 'FINAL') for i in range(390)])
jobs=[dict(index=i,job=c['jobs'][i],worker=dict(status='STOPPED_OR_FAILED' if i in RECOVER else 'PASS',
returncode=-9 if i in RECOVER else 0,cpu_seconds=2. if i in RECOVER else 1.)) for i in range(235)]
old=dict(status='S3A_STOPPED_INCOMPLETE',stop_reason='S3A_COORDINATOR_INTERRUPTED',accounting_complete=True,
jobs=jobs,not_run_indices=NEVER,worker_cpu_seconds=239.)
scope=dict(recovery_indices=RECOVER,never_attempted_indices=NEVER,original_worker_CPU_s=239.)
return c,old,scope
class Recovery(unittest.TestCase):
def test_exact_159_partition(self):
c,old,s=fixture();passed=validate_scope(c,old,s)
self.assertEqual(len(passed),231);self.assertEqual(len(ORDER),159)
self.assertFalse(set(r['index'] for r in passed)&set(ORDER))
self.assertEqual({r['index'] for r in passed}|set(ORDER),set(range(390)))
def test_changed_scope_or_spec_rejected(self):
for field in ('recovery_indices','never_attempted_indices'):
c,old,s=fixture();s[field]=s[field]+[1]
with self.assertRaises(ValueError):validate_scope(c,old,s)
c,old,s=fixture();old['jobs'][0]['job']=dict(index=999)
with self.assertRaises(ValueError):validate_scope(c,old,s)
def test_prior_charge_once_including_aborted(self):
c,old,s=fixture();p=validate_scope(c,old,s)
self.assertEqual(canonical_accounting(old,p),239.)
new=copy.deepcopy(old['jobs'][230]);new['worker']=dict(status='PASS',returncode=0,cpu_seconds=7.)
self.assertEqual(canonical_accounting(old,p+[new]),246.)
def test_original_success_cannot_change_or_repeat(self):
c,old,s=fixture();p=validate_scope(c,old,s)
with self.assertRaises(ValueError):canonical_accounting(old,p+[p[0]])
p=copy.deepcopy(p);p[0]['worker']['cpu_seconds']=0
with self.assertRaises(ValueError):canonical_accounting(old,p)
def test_selection_after_360_not_390(self):
rows=[dict(index=i) for i in range(360)]
self.assertTrue(selection_ready(rows,{}))
self.assertFalse(selection_ready(rows,{0:'active'}))
self.assertFalse(selection_ready(rows[:-1],{}))
self.assertEqual([i for i in ORDER if i<360],RECOVER+list(range(235,360)))
self.assertEqual(ORDER[-30:],list(range(360,390)))
def test_incomplete_original_ledger_rejected(self):
c,old,s=fixture();old['worker_cpu_seconds']=1
with self.assertRaises(ValueError):validate_scope(c,old,s)
if __name__=='__main__':unittest.main()