"""B8 metadata only: strict HTTP ranges; never accept a full ZIP response.""" import io, json, time, urllib.request, zipfile, hashlib from pathlib import Path ROOT = Path(__file__).parent URL = 'https://zenodo.org/api/records/10523313/files/data.zip/content' SIZE = 41368619333 calls = [] class Remote(io.RawIOBase): def __init__(self): self.pos = 0 def seekable(self): return True def readable(self): return True def tell(self): return self.pos def seek(self, off, whence=0): self.pos = off if whence == 0 else self.pos+off if whence == 1 else SIZE+off if not 0 <= self.pos <= SIZE: raise ValueError('seek out of range') return self.pos def read(self, n=-1): n = SIZE-self.pos if n < 0 else min(n, SIZE-self.pos) if not n: return b'' if n > 2**20: raise ValueError('metadata request exceeds 1MiB') lo, hi = self.pos, self.pos+n-1 req=urllib.request.Request(URL, headers={'Range': f'bytes={lo}-{hi}', 'Accept-Encoding':'identity'}) with urllib.request.urlopen(req, timeout=25) as r: if r.status != 206 or r.headers.get('Content-Range') != f'bytes {lo}-{hi}/{SIZE}': raise ValueError('server did not honor exact Range; no payload read') data=r.read(n+1) if len(data)!=n: raise ValueError('range length differs') calls.append(dict(start=lo,end=hi,bytes=n,sha256=hashlib.sha256(data).hexdigest())) self.pos+=n return data start=time.process_time() try: with zipfile.ZipFile(Remote()) as z: rows=[dict(name=i.filename,bytes=i.file_size,compressed_bytes=i.compress_size, header_offset=i.header_offset,crc32=i.CRC) for i in z.infolist()] result=dict(status='DIRECTORY_READ',url=URL,archive_bytes=SIZE,members=rows, range_calls=calls,bytes_received=sum(c['bytes'] for c in calls), cpu_seconds=time.process_time()-start,member_payloads_read=0) except Exception as e: result=dict(status='FAILED',error=repr(e),range_calls=calls,cpu_seconds=time.process_time()-start) (ROOT/'v2-central-directory.json').write_text(json.dumps(result,indent=2)+'\n') print(json.dumps({k:v for k,v in result.items() if k!='members'}))