Files
MeltMind/artifacts/phase2-ml-planning-2026-11-17/read_v2_directory.py

45 lines
2.2 KiB
Python

"""B8 metadata only: strict HTTP ranges; never accept a full ZIP response."""
import io, json, time, urllib.request, zipfile, hashlib
from pathlib import Path
ROOT = Path(__file__).parent
URL = 'https://zenodo.org/api/records/10523313/files/data.zip/content'
SIZE = 41368619333
calls = []
class Remote(io.RawIOBase):
def __init__(self): self.pos = 0
def seekable(self): return True
def readable(self): return True
def tell(self): return self.pos
def seek(self, off, whence=0):
self.pos = off if whence == 0 else self.pos+off if whence == 1 else SIZE+off
if not 0 <= self.pos <= SIZE: raise ValueError('seek out of range')
return self.pos
def read(self, n=-1):
n = SIZE-self.pos if n < 0 else min(n, SIZE-self.pos)
if not n: return b''
if n > 2**20: raise ValueError('metadata request exceeds 1MiB')
lo, hi = self.pos, self.pos+n-1
req=urllib.request.Request(URL, headers={'Range': f'bytes={lo}-{hi}', 'Accept-Encoding':'identity'})
with urllib.request.urlopen(req, timeout=25) as r:
if r.status != 206 or r.headers.get('Content-Range') != f'bytes {lo}-{hi}/{SIZE}':
raise ValueError('server did not honor exact Range; no payload read')
data=r.read(n+1)
if len(data)!=n: raise ValueError('range length differs')
calls.append(dict(start=lo,end=hi,bytes=n,sha256=hashlib.sha256(data).hexdigest()))
self.pos+=n
return data
start=time.process_time()
try:
with zipfile.ZipFile(Remote()) as z:
rows=[dict(name=i.filename,bytes=i.file_size,compressed_bytes=i.compress_size,
header_offset=i.header_offset,crc32=i.CRC) for i in z.infolist()]
result=dict(status='DIRECTORY_READ',url=URL,archive_bytes=SIZE,members=rows,
range_calls=calls,bytes_received=sum(c['bytes'] for c in calls),
cpu_seconds=time.process_time()-start,member_payloads_read=0)
except Exception as e:
result=dict(status='FAILED',error=repr(e),range_calls=calls,cpu_seconds=time.process_time()-start)
(ROOT/'v2-central-directory.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps({k:v for k,v in result.items() if k!='members'}))