Preserve item1 terminal results and freeze seed1 training review entry
This commit is contained in:
1 parent
888f706612
commit
c0f89a5e95
114 files changed
+282280
No files matched your search
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"timestamp": "2026-11-18T06:26:28.314006+00:00",
|
||||
"binding_sha256": "a1787bfcce696ca46df245694295f3fbc270da355d9670835296ba447614edc8",
|
||||
"code_files_checked": 30,
|
||||
"source_files_checked": 17,
|
||||
"original_training_code_pins_checked": 23,
|
||||
"implementation_files_current_and_local_git_checked": 35,
|
||||
"A_reference_metadata_matches": 1537,
|
||||
"unique_cases": 586,
|
||||
"per_split_counts": [
|
||||
304,
|
||||
311,
|
||||
308,
|
||||
309,
|
||||
305
|
||||
],
|
||||
"A_prediction_units": "seconds",
|
||||
"unit_evidence": [
|
||||
"Frozen A predictions-freeze.json prediction_units=seconds",
|
||||
"tools/gre24_score.py:53 multiplies normalized predictions by t_scale_seconds before saving",
|
||||
"tools/track_a_postfit_worker.py uses seconds for H2 and cached truth"
|
||||
],
|
||||
"remote_binding_semantically_equal": true,
|
||||
"remote_read_limitation": "Rendered extraction compared as JSON; no remote all-file raw-byte hash claim",
|
||||
"remote_read_sha256": "380db300857e92360d9d8a1b239b234bdb0d226b78b8028d5b04927d99d93f77",
|
||||
"root_markers_at_audit": {
|
||||
"review.json": false,
|
||||
"started.json": false,
|
||||
"complete.json": false,
|
||||
"models-adopted.json": false,
|
||||
"predictions-manifest.json": false
|
||||
},
|
||||
"independent_synthetic_tests": {
|
||||
"tests": 15,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"status": "PASS",
|
||||
"wall_seconds": 0.012,
|
||||
"provenance": "Executed earlier in this same review before interruption; exec session 48233 exited 0; not rerun"
|
||||
},
|
||||
"author_acceptance": {
|
||||
"tests": 15,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 3.435988618,
|
||||
"wall_seconds": 3.4529706729808822,
|
||||
"scientific_arrays_read": false,
|
||||
"targets_read": false,
|
||||
"gpu_used": false
|
||||
},
|
||||
"scientific_arrays_loaded": false,
|
||||
"targets_loaded": false,
|
||||
"models_loaded": false,
|
||||
"predictions_executed": false,
|
||||
"git_writes": false
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"url": "https://gitea.frank14f.online/Frank14f/MeltMind/raw/commit/888f7066/home-runs/track-a-postfit-2026-11-18/expected-review-binding.json",
|
||||
"extract_id": "extract_940b3e8baae7ef218a6e091dbd47d687",
|
||||
"kind": "Rendered content semantic comparison only; not a raw remote byte hash",
|
||||
"binding": {
|
||||
"reviewed_commit": "b164071e391c92ad8ee99dfa96c37f5b17991731",
|
||||
"contract_sha256": "19faa4a9e14a19cedfeac3cbc5334462e10892f49269384d655449b525f6068e",
|
||||
"implementation_proof_sha256": "83e178efae084bd39994a22b21d2d020a9cb46f144ba11dbc6a4e260f1b46124",
|
||||
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit.py": "9526b6d2678e7a8d6935c240002a071b3fd6f1b385372695877997a96ee50c1f",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit.py": "976e1278aa0d8220501d2882436d8c039c878284cf9c64cf90f826886c13e260",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit.py": "8d9738bc15a622b069c7455e937a34ee1efdab64bc5685ee3350b81207bca6de",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_guard.py": "768415110ba43c9d2aa150ce0851562f9d6e00448364052ad6f29c3718236b14",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_worker.py": "dca3c127efaf75daab542b4d6b0e2828f59147ee45147de11f61e63ea4347855"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"reviewed_commit": "b164071e391c92ad8ee99dfa96c37f5b17991731",
|
||||
"contract_sha256": "19faa4a9e14a19cedfeac3cbc5334462e10892f49269384d655449b525f6068e",
|
||||
"implementation_proof_sha256": "83e178efae084bd39994a22b21d2d020a9cb46f144ba11dbc6a4e260f1b46124",
|
||||
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit.py": "9526b6d2678e7a8d6935c240002a071b3fd6f1b385372695877997a96ee50c1f",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit.py": "976e1278aa0d8220501d2882436d8c039c878284cf9c64cf90f826886c13e260",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit.py": "8d9738bc15a622b069c7455e937a34ee1efdab64bc5685ee3350b81207bca6de",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_guard.py": "768415110ba43c9d2aa150ce0851562f9d6e00448364052ad6f29c3718236b14",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_worker.py": "dca3c127efaf75daab542b4d6b0e2828f59147ee45147de11f61e63ea4347855"
|
||||
},
|
||||
"verdict": "PASS",
|
||||
"reviewer": "MeltAssist",
|
||||
"review_stage": "Independent Track A postfit pre-score implementation review",
|
||||
"timestamp": "2026-11-18T06:26:52.095885+00:00",
|
||||
"request": "1794982750",
|
||||
"evidence_path": "/home/frank14f/optane/MeltMind/artifacts/meltassist-track-a-postfit-review-2026-11-18/evidence.json",
|
||||
"evidence_sha256": "3d6fefbc6044aefe11252ad144558f0a97964a71a0f327159e8a5cd261d111ec",
|
||||
"publication_commit": "888f7066",
|
||||
"publication_is_not_implementation": true,
|
||||
"blocking_findings": [],
|
||||
"actual_read_scope": [
|
||||
"Seven new Track A postfit sources, relevant shared loader, review validator, full-node metric and original A prediction producer",
|
||||
"Exact contract, implementation proof, five-field binding, reuse entry, frozen original A prediction metadata and original training source pins",
|
||||
"Claude 1794982020.974879 scope/budget and 1794982202.809969 fixed descriptive estimator",
|
||||
"Remote 888f7066 binding rendered extraction; JSON semantically equal, not full remote raw-byte verification"
|
||||
],
|
||||
"checks": {
|
||||
"training_terminal_gate": "Five distinct PASS endpoints, guard receipt equality and integer-zero monitor_errors, 200 epochs/steps, frozen terminal/models, no live training process; actual adoption remains runtime",
|
||||
"prediction_gate": "Five serial GPU2 jobs; exact checkpoints and learning normalization; label-free full native test-node loader; no clipping/refit; 1537 visits/586 unique parts; old 14 failures retained",
|
||||
"units": "Frozen old A prediction metadata states seconds and producer converts before saving; H2 and truth also seconds",
|
||||
"custody_gate": "Exact new PASS frozen with binding and contract; all model/prediction/source barriers checked before target IO; one seconds load per unique part shared across its visits",
|
||||
"statistics": "Full-node RMSE/max(full truth), symmetric undefined support, per-split min/median/max and five-split mean; unique-part mean H2-A then equal-part bootstrap 2000 seed2026111408 percentile-linear 95%; candidate/defined visits retained; no significance gate",
|
||||
"resources": "162000 total GPU-s with prior training counted once; 7200 postfit GPU-s, 900 per serial prediction incl setup/idle, 3600 postfit CPU-s with reservations and serialized initial counters; 15GiB VRAM/16GiB RSS/20GiB root disk; no retry",
|
||||
"identity": "30 current code files, 17 sources, 23 original training code pins, 35 local immutable implementation files verified; all 1537 A reference records match frozen source metadata",
|
||||
"synthetic": "15 independent CPU synthetic checks PASS in same review, no scientific workloads"
|
||||
},
|
||||
"limitations": [
|
||||
"This PASS reviews the bound implementation; it is not observed training completion, model adoption, prediction completion or scientific result acceptance",
|
||||
"Runtime must still enforce all five frozen training terminals, exact review adoption, model identities, resource and prediction barriers before scoring",
|
||||
"Resource enforcement is sampled; overshoot and failures remain chargeable and cannot be silently retried",
|
||||
"Remote publication was checked through rendered binding JSON only; local fixed Git files were byte/hash checked",
|
||||
"Protocol-level re-creation with original author split IDs unavailable; random-part geometry-family leakage remains; no superiority claim from paper context",
|
||||
"No actual target, prediction-array or model values were loaded; original A binary hashes are enforced at runtime, while this review checked every frozen A metadata reference"
|
||||
],
|
||||
"scope": {
|
||||
"new_scientific_computation": false,
|
||||
"new_training": false,
|
||||
"new_prediction": false,
|
||||
"new_target_reads": false,
|
||||
"source_edits": false,
|
||||
"git_writes": false,
|
||||
"next_stage_authorized": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""CPU synthetic seed/role/cost/release/guard tests; no real arrays or GPU."""
|
||||
import ast,copy,inspect,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
from phase3_seed1_common import fold_view,project_cell,project_total,dispatchable,first_epoch_release,available_gpus,GPUS,ASSIGN
|
||||
from phase2_h2_data import load_part,standardization
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
import phase2_h2_s2_worker as original
|
||||
import phase3_seed1_worker as new
|
||||
import phase3_seed1_guard as guard
|
||||
|
||||
class SeedTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.pilot=dict(folds=[dict(fold=i,learning=['l'+str(i)],validation=['v'+str(i)],test=['t'+str(i)],learning_nodes=2) for i in range(5)],parts={'l'+str(i):dict(nodes=2) for i in range(5)})
|
||||
self.jobs=[dict(id=i,gpu_uuid=GPUS[ASSIGN[i]],gpu_seconds=22320,validation_seconds_bound=20) for i in range(5)]
|
||||
self.c=dict(jobs=self.jobs,gpu_cap=118800,postfit_gpu_reserve=7200,phase3_gpu_cap=612000,phase3_prior_gpu_once=73000)
|
||||
def test_all_five_original_roles(self):
|
||||
old=copy.deepcopy(self.pilot)
|
||||
for i in range(5):self.assertEqual(fold_view(self.pilot,i)['folds'][0],self.pilot['folds'][i])
|
||||
self.assertEqual(old,self.pilot)
|
||||
def test_foreign_and_overlap_rejected(self):
|
||||
for i in [True,-1,5]:
|
||||
with self.assertRaises(PermissionError):fold_view(self.pilot,i)
|
||||
self.pilot['folds'][0]['test']=['l0']
|
||||
with self.assertRaises(ValueError):fold_view(self.pilot,0)
|
||||
def test_test_role_never_used_for_train_or_normalization(self):
|
||||
for i in range(5):
|
||||
v=fold_view(self.pilot,i)
|
||||
for role in ['learning','validation']:
|
||||
with self.assertRaises(PermissionError):load_part(v,'t'+str(i),{},role,Path('/unused'))
|
||||
def features(c,case,name):
|
||||
self.assertEqual((case,name),('l'+str(i),'features'));return np.vstack([np.zeros(19),np.full(19,2.)])
|
||||
with tempfile.TemporaryDirectory() as d,patch('phase2_h2_data.cached',features):
|
||||
s=standardization(v,Path(d));self.assertEqual(s['parts'],['l'+str(i)]);self.assertEqual(s['mean'],[1.]*19)
|
||||
def test_epoch_math_identical_except_seed_and_cost_wait(self):
|
||||
def loop(fn):
|
||||
t=ast.parse(inspect.getsource(fn));n=next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch')
|
||||
n.body=[b for b in n.body if not(isinstance(b,ast.If) and ast.unparse(b.test)=='epoch == 1')]
|
||||
return n
|
||||
a,b=loop(original.run),loop(new.run)
|
||||
seq=[n for n in ast.walk(b) if isinstance(n,ast.Call) and ast.unparse(n.func)=='np.random.SeedSequence'];self.assertEqual(len(seq),1);self.assertEqual(seq[0].args[0].elts[0].value,1);seq[0].args[0].elts[0].value=0
|
||||
self.assertEqual(ast.dump(a),ast.dump(b))
|
||||
def test_all_rng_seed1_and_final_checkpoint(self):
|
||||
tree=ast.parse(inspect.getsource(new.run));calls={ast.unparse(n.func):n for n in ast.walk(tree) if isinstance(n,ast.Call)}
|
||||
for name in ['random.seed','np.random.seed','torch.manual_seed','torch.cuda.manual_seed_all']:self.assertEqual(calls[name].args[0].value,1)
|
||||
saves=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and ast.unparse(n.func)=='torch.save'];self.assertEqual(len(saves),1)
|
||||
fields={k.arg:ast.literal_eval(k.value) for k in saves[0].args[0].keywords if k.arg in ['seed','role']};self.assertEqual(fields,dict(seed=1,role='last_epoch_only'))
|
||||
def test_first_epoch_gate_not_all_folds_barrier(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);save(p/'contract.json',{});self.assertFalse(first_epoch_release(p,0));save(p/'jobs/0/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(p/'contract.json')));self.assertTrue(first_epoch_release(p,0));self.assertFalse(first_epoch_release(p,1));save(p/'global-stop.json',{})
|
||||
with self.assertRaises(RuntimeError):first_epoch_release(p,0)
|
||||
def test_full_reserves_then_settled_cost_once(self):
|
||||
self.assertEqual(project_total(self.c,{}, {})['projected_gpu_seconds'],118800)
|
||||
d=project_total(self.c,{}, {0:dict(gpu_seconds_charged=20000)})
|
||||
self.assertEqual(d['projected_gpu_seconds'],116480);self.assertEqual(d['projected_phase3_gpu_seconds'],189480)
|
||||
def test_validation_and_setup_in_projection(self):
|
||||
o=dict(epoch=1,charged=100,max_epoch_seconds=80,max_validation_seconds=30)
|
||||
d=project_cell(self.jobs[0],o);self.assertEqual(d['projected_gpu_seconds'],100+1.25*(199*80+20*30)+120)
|
||||
def test_boundary_overrun_stops(self):
|
||||
d=project_cell(self.jobs[0],dict(epoch=1,charged=100,max_epoch_seconds=100));self.assertFalse(d['within_cap'])
|
||||
self.assertEqual(project_total(self.c,{0:d},{})['status'],'STOP_BUDGET_PROJECTION')
|
||||
def test_parallel_dispatch_respects_held_gpus_and_identity(self):
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs,[],set(GPUS[:2]))],[0,1])
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[self.jobs[0],self.jobs[1]],set(GPUS))],[2,3])
|
||||
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[],{GPUS[0],GPUS[3]})],[3,4])
|
||||
def test_gpu2_waits_for_prior_track_postfit(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);c=dict(item1_recovery_root=str(r/'item'),track_a_training_root=str(r/'train'),track_a_postfit_root=str(r/'post'))
|
||||
with patch('psutil.process_iter',return_value=[]):
|
||||
self.assertEqual(available_gpus(c),{GPUS[0]});save(r/'item/complete.json',{});save(r/'train/complete.json',{})
|
||||
self.assertEqual(available_gpus(c),{GPUS[0],GPUS[1],GPUS[3]});save(r/'post/complete.json',{});self.assertEqual(available_gpus(c),set(GPUS))
|
||||
def guard_exercise(self,peak=100,monitor=0):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
q=Path(d);save(q/'contract.json',{});r=q/'jobs/0';r.mkdir(parents=True)
|
||||
j=dict(id=0,gpu_uuid='synthetic-seed1',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,target_steps=400,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)
|
||||
def work(*args,**kwargs):
|
||||
save(r/'worker/result.json',dict(status='COMPLETE',fold=0,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=monitor)
|
||||
with patch.object(guard,'config',return_value=(dict(jobs=[j]),{})),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',work),patch.object(guard.time,'sleep'):
|
||||
rc=guard.run(q,0)
|
||||
return rc,json.loads((r/'complete.json').read_text())
|
||||
def test_guard_clean_complete(self):self.assertEqual(self.guard_exercise()[0],0)
|
||||
def test_guard_rejects_monitor_errors(self):self.assertIn('WORKER_MONITOR_ERRORS',self.guard_exercise(monitor=1)[1]['reasons'])
|
||||
def test_guard_preserves_excess_recorded_peak(self):self.assertIn('RECORDED_VRAM_LIMIT',self.guard_exercise(peak=16*2**30)[1]['reasons'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"tests": 14,
|
||||
"failures": 3,
|
||||
"errors": 2,
|
||||
"test_body_seconds": 0.075,
|
||||
"exit_code": 1,
|
||||
"cause": "Test fixtures omitted parent directories before atomic save; first-epoch/availability and all three guard fixtures affected",
|
||||
"actual_scientific_execution": false
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"tests": 15,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 3.220114309,
|
||||
"wall_seconds": 3.228956536971964,
|
||||
"scientific_arrays_read": false,
|
||||
"gpu_used": false
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
test_all_five_original_roles (test_phase3_seed1.SeedTests) ... ok
|
||||
test_all_rng_seed1_and_final_checkpoint (test_phase3_seed1.SeedTests) ... ok
|
||||
test_assignment_is_hash_bound_and_fold_specific (test_phase3_seed1.SeedTests) ... ok
|
||||
test_boundary_overrun_stops (test_phase3_seed1.SeedTests) ... ok
|
||||
test_epoch_math_identical_except_seed_and_cost_wait (test_phase3_seed1.SeedTests) ... ok
|
||||
test_first_epoch_gate_not_all_folds_barrier (test_phase3_seed1.SeedTests) ... ok
|
||||
test_foreign_and_overlap_rejected (test_phase3_seed1.SeedTests) ... ok
|
||||
test_full_reserves_then_settled_cost_once (test_phase3_seed1.SeedTests) ... ok
|
||||
test_gpu2_waits_for_prior_track_postfit (test_phase3_seed1.SeedTests) ... ok
|
||||
test_guard_clean_complete (test_phase3_seed1.SeedTests) ... ok
|
||||
test_guard_preserves_excess_recorded_peak (test_phase3_seed1.SeedTests) ... ok
|
||||
test_guard_rejects_monitor_errors (test_phase3_seed1.SeedTests) ... ok
|
||||
test_parallel_dispatch_respects_held_gpus_and_identity (test_phase3_seed1.SeedTests) ... ok
|
||||
test_test_role_never_used_for_train_or_normalization (test_phase3_seed1.SeedTests) ... ok
|
||||
test_validation_and_setup_in_projection (test_phase3_seed1.SeedTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 15 tests in 0.076s
|
||||
|
||||
OK
|
||||
Reference in new issue
Block a user