diff --git a/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/contract.json b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/contract.json index b7b145cb9..e76cfdf7e 100644 --- a/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/contract.json +++ b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/contract.json @@ -43,10 +43,10 @@ "/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf", "/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447", "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", - "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_entry.py": "152719b82f949b7b13afb01f47a43dadae02776e2551689624b14df68e900151", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_entry.py": "e978c429f9bad89ff77f03a6ca448509159deb67e31ffebe6b40b61b2d80267b", "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_worker.py": "3fe8f1bdbec1e63ae01dcdf10fd66dead73afc5b827020e9b057414da1d29f90", - "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_custody.py": "b8c066b29c73fd3c5d2dd55c08d2f53885cd15095be90ae29df70c502b4c6b56", - "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_drive_acl.py": "6ecfded05844975575e1e98a26627808f306254949c9ed741accae31cfbb9bdc", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_custody.py": "7e5283ddd283f4f55a2dec7513042cc86c67e30e3e63ba96614945f8d8b8f276", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_greif_drive_acl.py": "2e5201adcd8c73c7c5efc1fe1197bf4f696e477b689ba5d0b3b69bd078c336b3", "/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741" }, "folds": { @@ -1290,13 +1290,13 @@ "freeze models", "form four target-free prediction sets", "freeze predictions", - "verified private Drive transfer", - "one controlled custody pass" + "one controlled local custody pass; independent optional verified private Drive transfer" ], "drive": { "purpose": "immutable model and prediction bundle", "git": "small SHA indexes and receipts only", - "acl_revision": "h-17e62811 live exhaustive Drive ACL and fixed owner-only inheritance chain" + "acl_revision": "h-c3d10e10 pre-upload parent ACL and post-upload file ACL", + "required_for_local_score": false }, "no_retry": true } diff --git a/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.json b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.json new file mode 100644 index 000000000..81106dd9a --- /dev/null +++ b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.json @@ -0,0 +1,295 @@ +{ + "status": "PASS", + "tests": 5, + "acl_cases": { + "normal_private_archive": { + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ], + [ + "fixture-archive", + null + ], + [ + "fixture-archive/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "fixture-archive", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ] + ], + "verified": true + }, + "after_missing-permissions": { + "further_uploads_blocked": true, + "evidence_retained": true, + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ], + [ + "fixture-archive", + null + ], + [ + "fixture-archive/permissions", + null + ] + ] + }, + "after_anyone-link-page2": { + "further_uploads_blocked": true, + "evidence_retained": true, + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ], + [ + "fixture-archive", + null + ], + [ + "fixture-archive/permissions", + null + ], + [ + "fixture-archive/permissions", + "page-two" + ] + ] + }, + "after_domain": { + "further_uploads_blocked": true, + "evidence_retained": true, + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ], + [ + "fixture-archive", + null + ], + [ + "fixture-archive/permissions", + null + ] + ] + }, + "after_inherited-outside": { + "further_uploads_blocked": true, + "evidence_retained": true, + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "root", + null + ], + [ + "fixture-archive", + null + ], + [ + "fixture-archive/permissions", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ] + ] + }, + "before_missing-permissions": { + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ] + ], + "upload_eligible": false + }, + "before_inherited-outside": { + "provider_queries": [ + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3", + null + ], + [ + "1quVPlAkM8lK95siVoA0YQ23I9I-0kch3/permissions", + null + ], + [ + "root", + null + ], + [ + "0AD1IMHKOPZB4Uk9PVA/permissions", + null + ] + ], + "upload_eligible": false + }, + "no_transport": { + "provider_queries": 0, + "upload_eligible": false + }, + "score_with_drive_unavailable": { + "synthetic_target_reads": 1, + "provider_queries": 0 + } + }, + "contract_sha256": "fce997026f6a3470acfe93a3e79a0efdc4d7c240f84891af1de0fd36e8573aa0", + "comparison_sha256": "9d42281db903a9fc268328e40d1f54c4a037f3cf58573e6fc78c9bb63ab3788e", + "original_contract_sha256": "f56e0d1a0e64fb46f2af6a731730308856083bdd18cfe030ab478b0cd841d539", + "original_review_binding_status": "BLOCKED", + "actual_gpu_queries": 0, + "actual_holdout_target_reads": 0, + "actual_uploads": 0, + "actual_permission_changes": 0, + "read_only_runtime_transport_configured": false, + "replay_log_sha256": "2cdec5da787b4e1a598b530723ff03a40960ba89457932ac548c8929f8e6331a" +} diff --git a/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.log b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.log new file mode 100644 index 000000000..a50b7a5e6 --- /dev/null +++ b/artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.log @@ -0,0 +1,10 @@ +test_complete_private_project_folder_and_archive_acl_pass_both_checks (test_track_b_gino_greif_drive_acl.CustodyACL) ... ok +test_missing_archive_evidence_link_only_and_domain_stop_further_uploads (test_track_b_gino_greif_drive_acl.CustodyACL) ... ok +test_missing_or_external_parent_permission_stops_before_upload (test_track_b_gino_greif_drive_acl.CustodyACL) ... ok +test_no_existing_transport_prevents_any_provider_attempt (test_track_b_gino_greif_drive_acl.CustodyACL) ... ok +test_scoring_with_no_drive_capability_or_receipts (test_track_b_gino_greif_drive_acl.CustodyACL) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 0.084s + +OK diff --git a/docs/track-b-gino-greif-drive-acl-review-2026-11-19.md b/docs/track-b-gino-greif-drive-acl-review-2026-11-19.md index 8fbe55fa7..1bdb2d305 100644 --- a/docs/track-b-gino-greif-drive-acl-review-2026-11-19.md +++ b/docs/track-b-gino-greif-drive-acl-review-2026-11-19.md @@ -1,9 +1,11 @@ -# GINO-Greif 保管权限差量复审 +# GINO-Greif 本地评分和 Drive 保管差量复审 -MeltAssist 在旧绑定 `f46f4036` 发现,只要本地回执写 `project_private=true`,缺权限或仅凭链接可读的文件都能到达留出读取。该回执现只用于绑定上传文件 ID、父目录和冻结归档,评分器从 Google Drive API 获取实际权限证据。 [原阻塞回执](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795092014928129) · [修订保管门](../tools/track_b_gino_greif_custody.py) +原 `f46f4036` 保管门接受本地 `project_private=true`,缺权限或仅凭链接可读时仍可进入留出读取。该绑定保留为 BLOCKED。按 Claude h-c3d10e10,评分的保管前置改回四模型和 424 份预测的本地冻结与哈希。Drive 归档另走自己的权限门。 [原阻塞回执](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795092014928129) · [最新裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795092045987099) -校验只涉及新归档、固定私有项目目录及其 My Drive 根。评分器走 HTTPS GET 读取 Google Drive v3 文件元数据及逐页权限,包括 published 视图。它核对原始二进制文件 SHA-256 和大小,核对父目录链、拥有者可见权限 ID 全集及前后文件版本,再保存实际权限证据。固定项目目录的既有只读元数据仅列已核项目拥有者,同一身份的归档及祖先 ACL 才能通过。任何 `anyone` 权限都拒绝,含 `allowFileDiscovery=false` 的凭链接访问。组、域、公开视图、项目外用户、无法核对的上级目录或中途权限变化也拒绝。 [Live ACL 实现](../tools/track_b_gino_greif_drive_acl.py) · [Drive 权限和继承](https://developers.google.com/workspace/drive/api/guides/manage-sharing) · [逐页权限 API](https://developers.google.com/workspace/drive/api/reference/rest/v3/permissions/list) +`score` 保留原精确启动许可,以及四模型、全部原节点预测、冻结清单和评分源校验。锁定权重与预测索引哈希后,原 212 件目标各读一次供两个种子评分。没有 Drive 客户端或 Drive 回执时,本地评分仍走同一条路径。科学合同、预算、训练入口和预登记比较内容沿用原绑定。 [本地评分](../tools/track_b_gino_greif_custody.py) -CPU 回放走了生产 `score()` 入口,在网络一层提供合成 API 响应。缺有效权限数据、后续页出现凭链接读权限、继承来的项目外用户及错误目录均在创建保管目录前拒绝,目标读为零。项目内权限对照先固定证据哈希,再到达合成目标读取一次。保留两套原 CPU 回放,共 18 项通过。预登记比较合同逐字一致,模型、训练、预算和评分定义沿原绑定。 [回放回执](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-replay.json) +`drive-preflight` 先核用户固定项目目录及其 My Drive 根的有效 ACL,确认权限均落在已核项目拥有者范围且前后版本一致,才产生归档前的目录证据。该证据只记录权限核验。后续实际上传仍按相应授权执行。上传后的 `drive-readback` 核对真实归档文件的 SHA-256、字节数、父目录,以及归档和上级链完整的有效 ACL。权限逐页读取,包含公开视图。任何 `anyone` 权限都拒绝,包括 `allowFileDiscovery=false` 的凭链接读取。域、组、继承的项目外用户和缺证均触发 HOLD。上传后的失败保留回执并阻断再开上传。现有文件和 ACL 均保留,可供审计。 [Drive 归档门](../tools/track_b_gino_greif_drive_acl.py) · [Drive 权限和继承](https://developers.google.com/workspace/drive/api/guides/manage-sharing) · [逐页权限 API](https://developers.google.com/workspace/drive/api/reference/rest/v3/permissions/list) -当前连接器能返回经裁剪的目录权限视图,权限 ID 和继承细节没有随响应提供。当前 Horsea 运行时未配置已授权的原始只读权限查询通道。生产路径只用已有授权凭据提供的只读传输,缺此通道即停在保管门。合成验收不能替代实际项目权限放行。本次仅读取固定项目目录与直接祖先的元数据,无权限更新、上传、留出目标读取和 GPU 查询。 [Drive 只读元数据说明](https://developers.google.com/workspace/drive/api/guides/file-metadata) +5 项差量 CPU 回放通过。无 Drive 通道或回执的评分对照到达一次合成目标读取。上级缺证或继承项目外授权时停在上传前。上传后缺证、仅凭链接可读、域授权或继承外部用户都固定 HOLD。完整项目范围合成对照通过两道 Drive 核验。原 13 项通过作为保留证据,本轮依裁定仅复验差量。比较合同逐字一致。本轮实际上传、权限修改、留出目标读取、GPU 查询和派发均为零。 [差量 CPU 回执](../artifacts/track-b-gino-greif-preparation-2026-11-19/combined-review-acl-v2/cpu-delta-replay.json) + +当前连接器返回裁剪后的权限视图,Horsea 尚无已授权的本地原始只读权限查询通道。故当前 Drive 归档保持 HOLD,本地评分门的 CPU 验收通过。部署的 Drive 核验路径只接受可提供完整只读权限清单的已有授权传输。 [Drive 只读元数据说明](https://developers.google.com/workspace/drive/api/guides/file-metadata) diff --git a/tests/test_track_b_gino_greif_drive_acl.py b/tests/test_track_b_gino_greif_drive_acl.py index a27f591d9..abf9e6ac7 100644 --- a/tests/test_track_b_gino_greif_drive_acl.py +++ b/tests/test_track_b_gino_greif_drive_acl.py @@ -1,6 +1,6 @@ -"""Exercise the real Greif custody gate with synthetic Drive API metadata.""" +"""CPU replay of independent local scoring and the complete Drive gate.""" -from contextlib import ExitStack +from contextlib import contextmanager, ExitStack import hashlib from pathlib import Path from tempfile import TemporaryDirectory @@ -16,123 +16,162 @@ from track_b_gino_shapenet_entry import read, save, sha EVIDENCE = {} OWNER_EMAIL = 'fixture-owner@example.invalid' ARCHIVE_ID = 'fixture-archive' +CASE = 'synthetic_part' +PROJECT = dict(private_drive_folder_id=acl.PROJECT_FOLDER) + + +@contextmanager +def replay(): + with TemporaryDirectory() as folder, ExitStack() as stack: + root = Path(folder) + for name in ('contract.json','models-frozen.json','predictions-frozen.json'): + save(root/name,{}) + archive = root/'gino-greif-models-and-predictions.zip' + archive.write_bytes(b'synthetic immutable archive') + digest, size = sha(archive), archive.stat().st_size + bundle = dict(path=str(archive),sha256=digest,bytes=size,contract_sha256=sha(root/'contract.json'), + model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json')) + save(root/'drive-upload-index.json',bundle) + compare, reference = root/'comparison.json', root/'reference.json' + save(reference,dict(status='COMPLETE',rows=[dict(case=CASE)]+[ + dict(case=f'other-{i}') for i in range(551)])) + save(compare,dict(roster=[dict(case=CASE,fold=0)],h2_score=dict(path=str(reference),sha256=sha(reference)))) + pilot, query = root/'pilot.json', root/'query.json' + save(pilot,{}) + save(query,dict(rows=[dict(case=CASE,included=True)])) + pins={str(p):sha(p) for p in (pilot,query)} + c=dict(budgets={'score_cpu_seconds':3600},comparison_contract=str(compare),comparison_contract_sha256=sha(compare)) + forecasts=[dict(case=CASE,fold=0,seed=s) for s in (0,1)] + owner=dict(id='fixture-owner-id',type='user',role='owner',emailAddress=OWNER_EMAIL) + outside=dict(id='foreign-id',type='user',role='reader',emailAddress='foreign@example.invalid', + permissionDetails=[dict(inherited=True,inheritedFrom=acl.PROJECT_MY_DRIVE_ROOT)]) + linked=dict(id='anyone',type='anyone',role='reader',allowFileDiscovery=False) + queried=[] + state=dict(before='project',after='project') + + def provider(path, args, token): + assert token=='fixture-token' + queried.append((path,args.get('pageToken'))) + after_upload=(root/'drive-readback.json').exists() + mode=state['after' if after_upload else 'before'] + if path.endswith('/permissions'): + assert args['includePermissionsForView']=='published' + if mode=='missing-permissions': + return {} + if mode=='anyone-link-page2' and path==ARCHIVE_ID+'/permissions': + if args.get('pageToken')=='page-two': + return dict(permissions=[linked]) + return dict(permissions=[owner],nextPageToken='page-two') + if mode=='inherited-outside' and path==acl.PROJECT_MY_DRIVE_ROOT+'/permissions': + return dict(permissions=[owner,outside]) + if mode=='domain' and path==ARCHIVE_ID+'/permissions': + return dict(permissions=[owner,dict(id='domain',type='domain',role='reader')]) + return dict(permissions=[owner]) + item=dict(id=path,ownedByMe=True,trashed=False,version='1',permissionIds=[owner['id']],mimeType=acl.FOLDER_TYPE) + if path==ARCHIVE_ID: + return dict(item,parents=[acl.PROJECT_FOLDER],mimeType='application/zip',size=str(size),sha256Checksum=digest) + if path==acl.PROJECT_FOLDER: + return dict(item,parents=[acl.PROJECT_MY_DRIVE_ROOT]) + if path=='root': + return dict(item,id=acl.PROJECT_MY_DRIVE_ROOT) + raise AssertionError('Unexpected item beyond the fixed project chain') + + stack.enter_context(patch.dict('os.environ',CUDA_VISIBLE_DEVICES='')) + custody.os.environ.pop(acl.TOKEN_FD,None) + stack.enter_context(patch.object(entry,'ROOT',root)) + # No Drive authority field is supplied to the local scoring path. + stack.enter_context(patch.object(entry,'authorize',return_value=({},0.))) + for k,v in dict(PILOT=pilot,QUERY=query,PINS=pins).items(): + stack.enter_context(patch.object(custody,k,v)) + stack.enter_context(patch.object(custody,'frozen',return_value=([],forecasts))) + stack.enter_context(patch.object(custody.resource,'setrlimit')) + stack.enter_context(patch.object(acl,'PROJECT_OWNER_SHA256',hashlib.sha256(OWNER_EMAIL.encode()).hexdigest())) + stack.enter_context(patch.object(acl,'_get_json',side_effect=provider)) + stack.enter_context(patch.object(custody,'summary',return_value={})) + targets=stack.enter_context(patch.object(custody,'target_array',return_value='synthetic target')) + def read_case(*_): + start=read(root/'score/custody-start.json') + assert start['model_manifest_sha256']==sha(root/'models-frozen.json') + assert start['manifest_sha256']==sha(root/'predictions-frozen.json') + targets({},CASE) + return {'case':CASE} + stack.enter_context(patch.object(custody,'one_case',side_effect=read_case)) + + def mark_uploaded(): + # The legacy assertion is intentionally still present and is ignored. + save(root/'drive-readback.json',dict(sha256Checksum=digest,size=str(size),parent_id=acl.PROJECT_FOLDER, + file_id=ARCHIVE_ID,project_private=True,contract_sha256=bundle['contract_sha256'])) + + yield root,c,state,queried,targets,mark_uploaded class CustodyACL(unittest.TestCase): - def replay(self, mode): - with TemporaryDirectory() as folder, ExitStack() as stack: - root = Path(folder) - for name in ('contract.json','models-frozen.json','predictions-frozen.json'): - save(root/name, {}) - archive = root/'gino-greif-models-and-predictions.zip' - archive.write_bytes(b'synthetic immutable archive') - digest, size = sha(archive), archive.stat().st_size - scope = dict(private_drive_folder_id=acl.PROJECT_FOLDER) - if mode == 'other-folder': - scope['private_drive_folder_id'] = 'out-of-scope' - bundle = dict(path=str(archive), sha256=digest, bytes=size,contract_sha256=sha(root/'contract.json'), - model_index_sha256=sha(root/'models-frozen.json'), - prediction_index_sha256=sha(root/'predictions-frozen.json')) - save(root/'drive-upload-index.json',bundle) - # This is the assertion that formerly bypassed all permission checks. - save(root/'drive-readback.json',dict(sha256Checksum=digest,size=str(size), - parent_id=scope['private_drive_folder_id'],file_id=ARCHIVE_ID,project_private=True, - contract_sha256=bundle['contract_sha256'])) - case = 'synthetic_part' - compare = root/'comparison.json' - reference = root/'reference.json' - save(reference,dict(status='COMPLETE',rows=[dict(case=case)]+[ - dict(case=f'other-{i}') for i in range(551)])) - save(compare,dict(roster=[dict(case=case,fold=0)],h2_score=dict(path=str(reference),sha256=sha(reference)))) - pilot, query = root/'pilot.json',root/'query.json' - save(pilot,{}) - save(query,dict(rows=[dict(case=case,included=True)])) - pins = {str(p):sha(p) for p in (pilot,query)} - c=dict(budgets={'score_cpu_seconds':3600},comparison_contract=str(compare),comparison_contract_sha256=sha(compare)) - forecasts=[dict(case=case,fold=0,seed=s) for s in (0,1)] - owner=dict(id='fixture-owner-id',type='user',role='owner',emailAddress=OWNER_EMAIL) - inherited=dict(id='foreign-id',type='user',role='reader',emailAddress='foreign@example.invalid', - permissionDetails=[dict(inherited=True,inheritedFrom=acl.PROJECT_MY_DRIVE_ROOT)]) - linked=dict(id='anyone',type='anyone',role='reader',allowFileDiscovery=False) - calls=[] - def provider(path, query, token): - self.assertEqual(token,'fixture-token') - calls.append((path, query.get('pageToken'))) - if path.endswith('/permissions'): - self.assertEqual(query['includePermissionsForView'],'published') - if mode=='missing-acl' and path==ARCHIVE_ID+'/permissions': - return {} - if mode=='anyone-link-page2' and path==ARCHIVE_ID+'/permissions': - if query.get('pageToken')=='fixture-page-2': - return dict(permissions=[linked]) - return dict(permissions=[owner],nextPageToken='fixture-page-2') - if mode=='inherited-outside' and path==acl.PROJECT_MY_DRIVE_ROOT+'/permissions': - return dict(permissions=[owner,inherited]) - return dict(permissions=[owner]) - item=dict(id=path,ownedByMe=True,trashed=False,version='1',permissionIds=[owner['id']],mimeType=acl.FOLDER_TYPE) - if path==ARCHIVE_ID: - return dict(item,parents=[acl.PROJECT_FOLDER],mimeType='application/zip',size=str(size),sha256Checksum=digest) - if path==acl.PROJECT_FOLDER: - return dict(item,parents=[acl.PROJECT_MY_DRIVE_ROOT]) - if path=='root': - return dict(item,id=acl.PROJECT_MY_DRIVE_ROOT) - raise AssertionError('Unexpected item outside the fixed chain') + def test_scoring_with_no_drive_capability_or_receipts(self): + with replay() as (root,c,_,queried,targets,_): + self.assertEqual(custody.score(c),0) + targets.assert_called_once() + self.assertEqual(queried,[]) + self.assertFalse((root/'drive-preflight.json').exists()) + EVIDENCE['score_with_drive_unavailable']=dict(synthetic_target_reads=1,provider_queries=0) - stack.enter_context(patch.dict('os.environ',CUDA_VISIBLE_DEVICES='')) - stack.enter_context(patch.object(entry,'ROOT',root)) - stack.enter_context(patch.object(entry,'authorize',return_value=(scope,0.))) - for k,v in dict(PILOT=pilot,QUERY=query,PINS=pins).items(): - stack.enter_context(patch.object(custody,k,v)) - stack.enter_context(patch.object(custody,'frozen',return_value=([],forecasts))) - stack.enter_context(patch.object(custody.resource,'setrlimit')) - stack.enter_context(patch.object(acl,'PROJECT_OWNER_SHA256',hashlib.sha256(OWNER_EMAIL.encode()).hexdigest())) - if mode=='no-transport': - custody.os.environ.pop(acl.TOKEN_FD,None) - else: - stack.enter_context(patch.object(acl,'_credential',return_value='fixture-token')) - stack.enter_context(patch.object(acl,'_get_json',side_effect=provider)) - stack.enter_context(patch.object(custody,'summary',return_value={})) - read_target=stack.enter_context(patch.object(custody,'target_array',return_value='synthetic target')) + def test_missing_or_external_parent_permission_stops_before_upload(self): + for mode in ('missing-permissions','inherited-outside'): + with self.subTest(mode=mode), replay() as (root,c,state,queried,targets,_): + state['before']=mode + with patch.object(acl,'_credential',return_value='fixture-token'): + self.assertEqual(custody.drive_preflight(c,PROJECT),1) + self.assertEqual(read(root/'drive-preflight.json')['status'],'HOLD_FURTHER_DRIVE_UPLOADS') + self.assertEqual(read(root/'drive-preflight-hold.json')['stage'],'BEFORE_UPLOAD') + self.assertFalse((root/'drive-readback.json').exists()) + targets.assert_not_called() + EVIDENCE['before_'+mode]=dict(provider_queries=queried,upload_eligible=False) - def case_reached(*_): - proof=root/'score/drive-acl-evidence.json' - self.assertEqual(read(root/'score/custody-start.json')['drive_acl_evidence_sha256'],sha(proof)) - read_target({},case) - return dict(case=case) + def test_missing_archive_evidence_link_only_and_domain_stop_further_uploads(self): + for mode in ('missing-permissions','anyone-link-page2','domain','inherited-outside'): + with self.subTest(mode=mode), replay() as (root,c,state,queried,targets,mark_uploaded): + with patch.object(acl,'_credential',return_value='fixture-token'): + self.assertEqual(custody.drive_preflight(c,PROJECT),0) + state['after']=mode + mark_uploaded() + self.assertEqual(custody.drive_readback(c,PROJECT),1) + count=len(queried) + with self.assertRaises(PermissionError): + custody.drive_preflight(c,PROJECT) + self.assertEqual(len(queried),count) + if mode=='anyone-link-page2': + self.assertIn((ARCHIVE_ID+'/permissions','page-two'),queried) + self.assertTrue((root/'drive-readback.json').exists()) + self.assertEqual(read(root/'drive-upload-hold.json')['stage'],'AFTER_UPLOAD') + self.assertFalse((root/'drive-upload-verified.json').exists()) + targets.assert_not_called() + EVIDENCE['after_'+mode]=dict(further_uploads_blocked=True,evidence_retained=True, + provider_queries=queried) - reader=stack.enter_context(patch.object(custody,'one_case',side_effect=case_reached)) - if mode=='project-only': - self.assertEqual(custody.score(c),0) - read_target.assert_called_once() - proof=read(root/'score/drive-acl-evidence.json') - self.assertEqual([x['file_id'] for x in proof['chain']], - [ARCHIVE_ID,acl.PROJECT_FOLDER,acl.PROJECT_MY_DRIVE_ROOT]) - EVIDENCE[mode] = dict(custody_reached=True,synthetic_reads=1,provider_queries=calls) - else: - with self.assertRaises(PermissionError): - custody.score(c) - reader.assert_not_called() - read_target.assert_not_called() - self.assertFalse((root/'score').exists()) - EVIDENCE[mode] = dict(custody_reached=False,synthetic_reads=0,provider_queries=calls) + def test_complete_private_project_folder_and_archive_acl_pass_both_checks(self): + with replay() as (root,c,_,queried,targets,mark_uploaded): + with patch.object(acl,'_credential',return_value='fixture-token'): + self.assertEqual(custody.drive_preflight(c,PROJECT),0) + before=read(root/'drive-preflight.json') + self.assertTrue(before['does_not_authorize_upload']) + self.assertEqual([row['file_id'] for row in before['drive']['chain']], + [acl.PROJECT_FOLDER,acl.PROJECT_MY_DRIVE_ROOT]) + mark_uploaded() + self.assertEqual(custody.drive_readback(c,PROJECT),0) + evidence=read(root/'drive-upload-verified.json') + self.assertEqual(evidence['status'],'PRIVATE_DRIVE_ARCHIVE_VERIFIED') + self.assertEqual([r['file_id'] for r in evidence['chain']], + [ARCHIVE_ID,acl.PROJECT_FOLDER,acl.PROJECT_MY_DRIVE_ROOT]) + self.assertEqual(evidence['sha256'],sha(root/'gino-greif-models-and-predictions.zip')) + targets.assert_not_called() + EVIDENCE['normal_private_archive']=dict(provider_queries=queried,verified=True) - def test_missing_current_transport_and_missing_effective_acl_stop(self): - for mode in ('no-transport','missing-acl'): - self.replay(mode) - - def test_unlisted_link_reader_is_blocked_even_on_later_page(self): - self.replay('anyone-link-page2') - self.assertIn((ARCHIVE_ID+'/permissions','fixture-page-2'), EVIDENCE['anyone-link-page2']['provider_queries']) - - def test_inherited_non_project_user_is_blocked(self): - self.replay('inherited-outside') - - def test_wrong_folder_is_rejected_before_provider_access(self): - self.replay('other-folder') - self.assertEqual(EVIDENCE['other-folder']['provider_queries'],[]) - - def test_project_acl_and_entire_ancestry_positive_control(self): - self.replay('project-only') + def test_no_existing_transport_prevents_any_provider_attempt(self): + with replay() as (root,c,_,queried,targets,_): + self.assertEqual(custody.drive_preflight(c,PROJECT),1) + self.assertEqual(queried,[]) + self.assertEqual(read(root/'drive-preflight-hold.json')['stage'],'BEFORE_UPLOAD') + targets.assert_not_called() + EVIDENCE['no_transport']=dict(provider_queries=0,upload_eligible=False) if __name__=='__main__': diff --git a/tools/track_b_gino_greif_custody.py b/tools/track_b_gino_greif_custody.py index dd07195be..f19d36f33 100644 --- a/tools/track_b_gino_greif_custody.py +++ b/tools/track_b_gino_greif_custody.py @@ -1,4 +1,4 @@ -"""Drive-backed Greif safekeeping and one guarded custody read per test case.""" +"""Local single-read Greif scoring and independently verified Drive safekeeping.""" from collections import defaultdict import hashlib @@ -16,7 +16,7 @@ from phase2_h2_data import cached from s2_custody_common import prediction_inputs from s2_custody_statistics import part_metrics import track_b_gino_greif_entry as entry -from track_b_gino_greif_drive_acl import prove as prove_drive_acl +from track_b_gino_greif_drive_acl import prove as prove_drive_acl, prove_folder from track_b_gino_greif_prepare import PILOT, QUERY, PINS from track_b_gino_shapenet_entry import BASE, checked, read, save, sha @@ -70,7 +70,7 @@ def pack(c): return 0 -def verify_drive(auth): +def local_bundle(): root=entry.ROOT bundle=read(root/'drive-upload-index.json') path=checked(bundle['path'],bundle['sha256']) @@ -80,6 +80,12 @@ def verify_drive(auth): or bundle['model_index_sha256']!=sha(root/'models-frozen.json') or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')): raise ValueError('Local archive no longer matches the fixed models and forecasts') + return bundle + + +def verify_drive(auth): + root=entry.ROOT + bundle=local_bundle() receipt=read(root/'drive-readback.json') # Transfer bookkeeping binds the file ID to the expected archive. The # actual ACL and remote file identity come from the live provider. @@ -91,6 +97,50 @@ def verify_drive(auth): return prove_drive_acl(receipt.get('file_id'),auth['private_drive_folder_id'],bundle['sha256'],bundle['bytes']) +def record_drive_hold(path, reason, scope): + if not path.exists(): + save(path,dict(status='HOLD_FURTHER_DRIVE_UPLOADS',stage=scope,reason=str(reason), + unix=time.time(),permissions_changed=False,files_deleted=False)) + + +def drive_preflight(c, auth): + root=entry.ROOT + if any((root/n).exists() for n in ('drive-upload-hold.json','drive-readback.json','drive-upload-verified.json')): + raise PermissionError('Existing Drive attempt or HOLD requires disposition before any further upload') + try: + frozen(c) + bundle=local_bundle() + evidence=prove_folder(auth.get('private_drive_folder_id')) + except (OSError, ValueError, KeyError) as exc: + record_drive_hold(root/'drive-preflight-hold.json',exc,'BEFORE_UPLOAD') + save(root/'drive-preflight.json',dict(status='HOLD_FURTHER_DRIVE_UPLOADS',unix=time.time())) + return 1 + save(root/'drive-preflight.json',dict(status='PROJECT_FOLDER_ACL_VERIFIED',unix=time.time(), + sha256=bundle['sha256'],contract_sha256=bundle['contract_sha256'],drive=evidence, + does_not_authorize_upload=True)) + return 0 + + +def drive_readback(c, auth): + root=entry.ROOT + if any((root/n).exists() for n in ('drive-upload-hold.json','drive-upload-verified.json')): + raise PermissionError('Existing Drive disposition is preserved') + try: + frozen(c) + bundle=local_bundle() + preflight=read(root/'drive-preflight.json') + if (preflight['status'],preflight['sha256'],preflight['contract_sha256'],preflight['drive']['parent_id']) != ( + 'PROJECT_FOLDER_ACL_VERIFIED',bundle['sha256'],bundle['contract_sha256'],auth.get('private_drive_folder_id')): + raise PermissionError('No matching pre-upload project-folder ACL evidence') + evidence=verify_drive(auth) + except (OSError, ValueError, KeyError) as exc: + record_drive_hold(root/'drive-upload-hold.json',exc,'AFTER_UPLOAD') + return 1 + save(root/'drive-upload-verified.json',dict(status='PRIVATE_DRIVE_ARCHIVE_VERIFIED',unix=time.time(), + **evidence, preflight_sha256=sha(root/'drive-preflight.json'))) + return 0 + + def bootstrap(rows, metric, fixed_support=None): values=defaultdict(list) unavailable=[] @@ -172,7 +222,7 @@ def one_case(pilot, meta, row, forecasts, reference, log): def score(c): - auth,_=entry.authorize(c) + entry.authorize(c) if os.environ.get('CUDA_VISIBLE_DEVICES')!='': raise PermissionError('Custody scoring must run with CUDA hidden') ceiling=resource.getrlimit(resource.RLIMIT_CPU)[1] @@ -181,7 +231,6 @@ def score(c): limit=min(limit,ceiling) resource.setrlimit(resource.RLIMIT_CPU,(limit,limit)) _,predictions=frozen(c) - drive=verify_drive(auth) compare=read(checked(c['comparison_contract'],c['comparison_contract_sha256'])) rosters={(r['fold'],r['case']) for r in compare['roster']} new={(r['fold'],r['seed'],r['case']):r for r in predictions} @@ -195,16 +244,13 @@ def score(c): query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']} out=entry.ROOT/'score' out.mkdir(exist_ok=False) - evidence=out/'drive-acl-evidence.json' - save(evidence,dict(drive,observed_before_target_reads=True)) - save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'], - drive_acl_evidence_sha256=sha(evidence), manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'), - before_test_target_reads=True)) + save(out/'custody-start.json',dict(unix=time.time(), + model_manifest_sha256=sha(entry.ROOT/'models-frozen.json'), + manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),before_test_target_reads=True)) results=[] with (out/'heldout-target-access.jsonl').open('x') as log: for row in compare['roster']: results.append(one_case(pilot,query[row['case']],row,new,reference,log)) save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results, - unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id'], - drive_acl_evidence_sha256=sha(evidence))) + unique_test_target_reads=len(results),no_new_decode=True)) return 0 diff --git a/tools/track_b_gino_greif_drive_acl.py b/tools/track_b_gino_greif_drive_acl.py index 5d4bb37a1..e416df89a 100644 --- a/tools/track_b_gino_greif_drive_acl.py +++ b/tools/track_b_gino_greif_drive_acl.py @@ -100,14 +100,9 @@ def _member_evidence(item, token): return matches -def prove(file_id, folder_id, digest, size): - if (folder_id != PROJECT_FOLDER or not isinstance(file_id, str) or not ID.fullmatch(file_id) - or file_id in (PROJECT_FOLDER, PROJECT_MY_DRIVE_ROOT, 'root')): - raise PermissionError('Archive must be in the fixed project folder') +def _prove_chain(lookup, archive=None): token = _credential() query = dict(fields=FILE_FIELDS, supportsAllDrives='true') - lookup = ((file_id, file_id, [PROJECT_FOLDER]), (PROJECT_FOLDER, PROJECT_FOLDER, [PROJECT_MY_DRIVE_ROOT]), - ('root', PROJECT_MY_DRIVE_ROOT, [])) observations = [] for locator, expected_id, parents in lookup: item = _get_json(locator, query, token) @@ -115,9 +110,9 @@ def prove(file_id, folder_id, digest, size): or item.get('ownedByMe') is not True or item.get('trashed') is not False or not isinstance(item.get('version'), str) or not item['version'].isdecimal()): raise PermissionError('Drive archive or inheritance chain does not match the fixed project') - if locator == file_id: + if archive and locator == archive['file_id']: if (item.get('mimeType') not in ('application/zip','application/x-zip','application/x-zip-compressed') - or (item.get('sha256Checksum'),item.get('size')) != (digest,str(size))): + or (item.get('sha256Checksum'),item.get('size')) != (archive['sha256'],str(archive['bytes']))): raise PermissionError('Authoritative Drive archive bytes do not match the frozen bundle') elif item.get('mimeType') != FOLDER_TYPE: raise PermissionError('Drive inheritance parent is not a folder') @@ -127,6 +122,24 @@ def prove(file_id, folder_id, digest, size): if _get_json(locator, query, token) != item: raise PermissionError('Drive metadata changed during ACL verification') return dict(source='live Google Drive v3 files.get and exhaustive permissions.list', - file_id=file_id,parent_id=folder_id,bytes=size,sha256=digest, chain=[dict(file_id=item['id'],version=item['version'],parents=item.get('parents', []), effective_permissions=permissions) for _,item,permissions in observations]) + + +def _fixed_parent_chain(folder_id): + if folder_id != PROJECT_FOLDER: + raise PermissionError('Drive destination must be the fixed project folder') + return ((PROJECT_FOLDER, PROJECT_FOLDER, [PROJECT_MY_DRIVE_ROOT]), ('root', PROJECT_MY_DRIVE_ROOT, [])) + + +def prove_folder(folder_id): + return dict(_prove_chain(_fixed_parent_chain(folder_id)), parent_id=folder_id) + + +def prove(file_id, folder_id, digest, size): + parents = _fixed_parent_chain(folder_id) + if (not isinstance(file_id, str) or not ID.fullmatch(file_id) + or file_id in (PROJECT_FOLDER, PROJECT_MY_DRIVE_ROOT, 'root')): + raise PermissionError('Archive must be in the fixed project folder') + archive = dict(file_id=file_id,parent_id=folder_id,bytes=size,sha256=digest) + return dict(_prove_chain(((file_id, file_id, [PROJECT_FOLDER]),) + parents, archive), **archive) diff --git a/tools/track_b_gino_greif_entry.py b/tools/track_b_gino_greif_entry.py index a763d1d1a..ed153d935 100644 --- a/tools/track_b_gino_greif_entry.py +++ b/tools/track_b_gino_greif_entry.py @@ -73,9 +73,10 @@ def expected_contract(): per_prediction=PRED_SECONDS,train_setup_reserve=FIT_SETUP_RESERVE,score_cpu_seconds=3600), run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits', 'freeze models','form four target-free prediction sets','freeze predictions', - 'verified private Drive transfer','one controlled custody pass'], + 'one controlled local custody pass; independent optional verified private Drive transfer'], drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only', - 'acl_revision':'h-17e62811 live exhaustive Drive ACL and fixed owner-only inheritance chain'}, + 'acl_revision':'h-c3d10e10 pre-upload parent ACL and post-upload file ACL', + 'required_for_local_score':False}, no_retry=True) @@ -92,8 +93,7 @@ def authorize(c): if (a.get('decision'),a.get('mode'),a.get('contract_sha256'),a.get('review_sha256')) != ( 'APPROVED',c['mode'],sha(ROOT/'contract.json'),sha(ROOT/'review.json')): raise PermissionError('Exact review plus separate stage authorization required') - if (not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id') - or not a.get('private_drive_folder_id')): + if not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id'): raise PermissionError('Direct user launch and ShapeNet verdict are required') verdict=read(checked(a['shape_verdict']['path'],a['shape_verdict']['sha256'])) if (verdict['contract_sha256']!=prep.reference.sha(prep.reference.ROOT/'contract.json') @@ -247,13 +247,17 @@ def run(c, auth, phase_booked): def main(): parser=argparse.ArgumentParser(description=__doc__) - parser.add_argument('action',choices=('run','pack','score')) + parser.add_argument('action',choices=('run','pack','score','drive-preflight','drive-readback')) args=parser.parse_args() c=contract() auth,booked=authorize(c) if args.action=='run': return run(c,auth,booked) - from track_b_gino_greif_custody import pack, score + from track_b_gino_greif_custody import drive_preflight, drive_readback, pack, score + if args.action in ('drive-preflight','drive-readback'): + with (ROOT/'drive-custody.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return drive_preflight(c,auth) if args.action=='drive-preflight' else drive_readback(c,auth) return pack(c) if args.action=='pack' else score(c)