Require authoritative Drive ACL and inheritance proof before Greif custody
This commit is contained in:
1 parent
ff3f1350c7
commit
dbb03c7dfa
10 files changed
+2876
-10
No files matched your search
@@ -16,6 +16,7 @@ from phase2_h2_data import cached
|
||||
from s2_custody_common import prediction_inputs
|
||||
from s2_custody_statistics import part_metrics
|
||||
import track_b_gino_greif_entry as entry
|
||||
from track_b_gino_greif_drive_acl import prove as prove_drive_acl
|
||||
from track_b_gino_greif_prepare import PILOT, QUERY, PINS
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
@@ -80,14 +81,14 @@ def verify_drive(auth):
|
||||
or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')):
|
||||
raise ValueError('Local archive no longer matches the fixed models and forecasts')
|
||||
receipt=read(root/'drive-readback.json')
|
||||
# Drive file metadata, recorded by the authorized connector from the private project folder.
|
||||
# Transfer bookkeeping binds the file ID to the expected archive. The
|
||||
# actual ACL and remote file identity come from the live provider.
|
||||
if (receipt.get('sha256Checksum'),receipt.get('size'),receipt.get('parent_id')) != (
|
||||
bundle['sha256'],str(bundle['bytes']),auth['private_drive_folder_id']):
|
||||
raise PermissionError('Private Drive readback SHA, size and folder are required before scoring')
|
||||
if (not receipt.get('file_id') or receipt.get('project_private') is not True
|
||||
or receipt.get('contract_sha256')!=bundle['contract_sha256']):
|
||||
raise PermissionError('Private Drive identity required before opening targets')
|
||||
return receipt
|
||||
if receipt.get('contract_sha256')!=bundle['contract_sha256']:
|
||||
raise PermissionError('Drive transfer does not match the reviewed contract')
|
||||
return prove_drive_acl(receipt.get('file_id'),auth['private_drive_folder_id'],bundle['sha256'],bundle['bytes'])
|
||||
|
||||
|
||||
def bootstrap(rows, metric, fixed_support=None):
|
||||
@@ -194,12 +195,16 @@ def score(c):
|
||||
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
|
||||
out=entry.ROOT/'score'
|
||||
out.mkdir(exist_ok=False)
|
||||
evidence=out/'drive-acl-evidence.json'
|
||||
save(evidence,dict(drive,observed_before_target_reads=True))
|
||||
save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'],
|
||||
manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),before_test_target_reads=True))
|
||||
drive_acl_evidence_sha256=sha(evidence), manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),
|
||||
before_test_target_reads=True))
|
||||
results=[]
|
||||
with (out/'heldout-target-access.jsonl').open('x') as log:
|
||||
for row in compare['roster']:
|
||||
results.append(one_case(pilot,query[row['case']],row,new,reference,log))
|
||||
save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results,
|
||||
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id']))
|
||||
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id'],
|
||||
drive_acl_evidence_sha256=sha(evidence)))
|
||||
return 0
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Prove the fixed project's effective ACL with live, owner-visible Drive metadata."""
|
||||
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
||||
PROJECT_FOLDER = '1quVPlAkM8lK95siVoA0YQ23I9I-0kch3'
|
||||
PROJECT_MY_DRIVE_ROOT = '0AD1IMHKOPZB4Uk9PVA'
|
||||
PROJECT_OWNER_SHA256 = 'feb12acc4cc98e55902b56268f726f3489d3e5baaf6aafef2fbff7e485f2a1b2'
|
||||
TOKEN_FD = 'MELTMIND_DRIVE_METADATA_TOKEN_FD'
|
||||
HOST = 'www.googleapis.com'
|
||||
FOLDER_TYPE = 'application/vnd.google-apps.folder'
|
||||
FILE_FIELDS = 'id,parents,driveId,mimeType,ownedByMe,trashed,permissionIds,version,size,sha256Checksum'
|
||||
ACL_FIELDS = ('nextPageToken,permissions(id,type,role,emailAddress,domain,allowFileDiscovery,deleted,view,'
|
||||
'permissionDetails(inherited,inheritedFrom,permissionType,role))')
|
||||
ID = re.compile(r'[A-Za-z0-9_-]+\Z')
|
||||
|
||||
|
||||
def _credential():
|
||||
fd = os.environ.get(TOKEN_FD, '')
|
||||
if not fd.isascii() or not fd.isdecimal() or int(fd) < 3:
|
||||
raise PermissionError('Existing authorized Drive metadata transport is unavailable')
|
||||
try:
|
||||
with os.fdopen(os.dup(int(fd)), 'rb') as source:
|
||||
token = source.readline(8193).strip().decode('ascii')
|
||||
except (OSError, ValueError, UnicodeError):
|
||||
raise PermissionError('Existing authorized Drive metadata transport is unavailable') from None
|
||||
if not token or len(token) > 8192 or any(not 33 <= ord(ch) <= 126 for ch in token):
|
||||
raise PermissionError('Drive metadata transport provided no usable credential')
|
||||
return token
|
||||
|
||||
|
||||
def _get_json(path, query, token):
|
||||
connection = http.client.HTTPSConnection(HOST, timeout=15)
|
||||
try:
|
||||
connection.request('GET', '/drive/v3/files/' + path + '?' + urlencode(query),
|
||||
headers={'Authorization': 'Bearer ' + token, 'Accept': 'application/json'})
|
||||
response = connection.getresponse()
|
||||
if response.status != 200:
|
||||
raise PermissionError(f'Authoritative Drive metadata status {response.status}')
|
||||
data = response.read(2**20 + 1)
|
||||
if len(data) > 2**20:
|
||||
raise PermissionError('Oversized Drive metadata')
|
||||
value = json.loads(data)
|
||||
if not isinstance(value, dict):
|
||||
raise PermissionError('Invalid Drive metadata response')
|
||||
return value
|
||||
except PermissionError:
|
||||
raise
|
||||
except (OSError, ValueError, http.client.HTTPException):
|
||||
raise PermissionError('Authoritative Drive metadata could not be verified') from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _listed_permissions(file_id, token):
|
||||
query = dict(fields=ACL_FIELDS, pageSize=100, supportsAllDrives='true', includePermissionsForView='published')
|
||||
rows, seen = [], set()
|
||||
for _ in range(20):
|
||||
page = _get_json(file_id + '/permissions', query, token)
|
||||
permissions = page.get('permissions')
|
||||
if not isinstance(permissions, list):
|
||||
raise PermissionError('Missing effective Drive permissions')
|
||||
rows.extend(permissions)
|
||||
if 'nextPageToken' not in page:
|
||||
return rows
|
||||
next_page = page['nextPageToken']
|
||||
if not isinstance(next_page, str) or not next_page or next_page in seen:
|
||||
raise PermissionError('Incomplete Drive permissions pagination')
|
||||
seen.add(next_page)
|
||||
query['pageToken'] = next_page
|
||||
raise PermissionError('Incomplete Drive permissions pagination')
|
||||
|
||||
|
||||
def _member_evidence(item, token):
|
||||
rows = _listed_permissions(item['id'], token)
|
||||
provider_ids = item.get('permissionIds')
|
||||
if not rows or not isinstance(provider_ids, list) or not all(isinstance(p, str) and p for p in provider_ids):
|
||||
raise PermissionError('No complete effective Drive permissions evidence')
|
||||
matches, ids = [], set()
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or row.get('type') != 'user' or row.get('view') == 'published':
|
||||
raise PermissionError('Public, linked, group or domain Drive access is out of scope')
|
||||
email, pid, role = row.get('emailAddress'), row.get('id'), row.get('role')
|
||||
if (not isinstance(email, str) or not isinstance(pid, str) or not pid or pid in ids
|
||||
or role not in ('owner', 'writer', 'commenter', 'reader') or row.get('deleted') is True):
|
||||
raise PermissionError('Unverifiable Drive principal or role')
|
||||
principal = hashlib.sha256(email.strip().casefold().encode()).hexdigest()
|
||||
if principal != PROJECT_OWNER_SHA256:
|
||||
raise PermissionError('Drive access outside the verified project principal')
|
||||
ids.add(pid)
|
||||
matches.append(dict(id=pid,type='user',role=role,principal_sha256=principal,
|
||||
permission_details=row.get('permissionDetails', [])))
|
||||
if ids != set(provider_ids) or not any(p['role']=='owner' for p in matches):
|
||||
raise PermissionError('Owner-visible effective Drive ACL is incomplete')
|
||||
return matches
|
||||
|
||||
|
||||
def prove(file_id, folder_id, digest, size):
|
||||
if (folder_id != PROJECT_FOLDER or not isinstance(file_id, str) or not ID.fullmatch(file_id)
|
||||
or file_id in (PROJECT_FOLDER, PROJECT_MY_DRIVE_ROOT, 'root')):
|
||||
raise PermissionError('Archive must be in the fixed project folder')
|
||||
token = _credential()
|
||||
query = dict(fields=FILE_FIELDS, supportsAllDrives='true')
|
||||
lookup = ((file_id, file_id, [PROJECT_FOLDER]), (PROJECT_FOLDER, PROJECT_FOLDER, [PROJECT_MY_DRIVE_ROOT]),
|
||||
('root', PROJECT_MY_DRIVE_ROOT, []))
|
||||
observations = []
|
||||
for locator, expected_id, parents in lookup:
|
||||
item = _get_json(locator, query, token)
|
||||
if (item.get('id') != expected_id or item.get('parents', []) != parents or item.get('driveId')
|
||||
or item.get('ownedByMe') is not True or item.get('trashed') is not False
|
||||
or not isinstance(item.get('version'), str) or not item['version'].isdecimal()):
|
||||
raise PermissionError('Drive archive or inheritance chain does not match the fixed project')
|
||||
if locator == file_id:
|
||||
if (item.get('mimeType') not in ('application/zip','application/x-zip','application/x-zip-compressed')
|
||||
or (item.get('sha256Checksum'),item.get('size')) != (digest,str(size))):
|
||||
raise PermissionError('Authoritative Drive archive bytes do not match the frozen bundle')
|
||||
elif item.get('mimeType') != FOLDER_TYPE:
|
||||
raise PermissionError('Drive inheritance parent is not a folder')
|
||||
observations.append((locator, item, _member_evidence(item, token)))
|
||||
# Require stable metadata across the three ACL reads, including each ancestor's version.
|
||||
for locator, item, _ in observations:
|
||||
if _get_json(locator, query, token) != item:
|
||||
raise PermissionError('Drive metadata changed during ACL verification')
|
||||
return dict(source='live Google Drive v3 files.get and exhaustive permissions.list',
|
||||
file_id=file_id,parent_id=folder_id,bytes=size,sha256=digest,
|
||||
chain=[dict(file_id=item['id'],version=item['version'],parents=item.get('parents', []),
|
||||
effective_permissions=permissions) for _,item,permissions in observations])
|
||||
@@ -15,7 +15,7 @@ from track_b_gino_formal_guard import guard
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = prep.ROOT / 'combined-review'
|
||||
ROOT = prep.ROOT / 'combined-review-acl-v2'
|
||||
SOURCE = Path(__file__).resolve().parent
|
||||
ADAPTER = prep.REVIEW_ROOT / 'preparation-binding.json'
|
||||
ADAPTER_SHA = '01d6581c33a6c26d1965acb13248bdb741de892e5929fff08a7e4af431f00d0b'
|
||||
@@ -61,6 +61,7 @@ def expected_contract():
|
||||
raise ValueError('Pre-registered Greif comparison differs')
|
||||
code=dict(base['code_sha256'],**c['code_sha256'])
|
||||
for name in ('track_b_gino_greif_entry.py','track_b_gino_greif_worker.py','track_b_gino_greif_custody.py',
|
||||
'track_b_gino_greif_drive_acl.py',
|
||||
'phase2_h2_model.py','phase2_h2_data.py','s2_custody_common.py','s2_custody_statistics.py'):
|
||||
code[str(SOURCE/name)] = sha(SOURCE/name)
|
||||
return dict(handoff='h-08c2a2a5',mode='gino-greif-combined',adapter_sha256=ADAPTER_SHA,
|
||||
@@ -73,7 +74,8 @@ def expected_contract():
|
||||
run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits',
|
||||
'freeze models','form four target-free prediction sets','freeze predictions',
|
||||
'verified private Drive transfer','one controlled custody pass'],
|
||||
drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only'},
|
||||
drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only',
|
||||
'acl_revision':'h-17e62811 live exhaustive Drive ACL and fixed owner-only inheritance chain'},
|
||||
no_retry=True)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user