45 lines
2.2 KiB
Python
45 lines
2.2 KiB
Python
"""B8 metadata only: strict HTTP ranges; never accept a full ZIP response."""
|
|
import io, json, time, urllib.request, zipfile, hashlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).parent
|
|
URL = 'https://zenodo.org/api/records/10523313/files/data.zip/content'
|
|
SIZE = 41368619333
|
|
calls = []
|
|
class Remote(io.RawIOBase):
|
|
def __init__(self): self.pos = 0
|
|
def seekable(self): return True
|
|
def readable(self): return True
|
|
def tell(self): return self.pos
|
|
def seek(self, off, whence=0):
|
|
self.pos = off if whence == 0 else self.pos+off if whence == 1 else SIZE+off
|
|
if not 0 <= self.pos <= SIZE: raise ValueError('seek out of range')
|
|
return self.pos
|
|
def read(self, n=-1):
|
|
n = SIZE-self.pos if n < 0 else min(n, SIZE-self.pos)
|
|
if not n: return b''
|
|
if n > 2**20: raise ValueError('metadata request exceeds 1MiB')
|
|
lo, hi = self.pos, self.pos+n-1
|
|
req=urllib.request.Request(URL, headers={'Range': f'bytes={lo}-{hi}', 'Accept-Encoding':'identity'})
|
|
with urllib.request.urlopen(req, timeout=25) as r:
|
|
if r.status != 206 or r.headers.get('Content-Range') != f'bytes {lo}-{hi}/{SIZE}':
|
|
raise ValueError('server did not honor exact Range; no payload read')
|
|
data=r.read(n+1)
|
|
if len(data)!=n: raise ValueError('range length differs')
|
|
calls.append(dict(start=lo,end=hi,bytes=n,sha256=hashlib.sha256(data).hexdigest()))
|
|
self.pos+=n
|
|
return data
|
|
|
|
start=time.process_time()
|
|
try:
|
|
with zipfile.ZipFile(Remote()) as z:
|
|
rows=[dict(name=i.filename,bytes=i.file_size,compressed_bytes=i.compress_size,
|
|
header_offset=i.header_offset,crc32=i.CRC) for i in z.infolist()]
|
|
result=dict(status='DIRECTORY_READ',url=URL,archive_bytes=SIZE,members=rows,
|
|
range_calls=calls,bytes_received=sum(c['bytes'] for c in calls),
|
|
cpu_seconds=time.process_time()-start,member_payloads_read=0)
|
|
except Exception as e:
|
|
result=dict(status='FAILED',error=repr(e),range_calls=calls,cpu_seconds=time.process_time()-start)
|
|
(ROOT/'v2-central-directory.json').write_text(json.dumps(result,indent=2)+'\n')
|
|
print(json.dumps({k:v for k,v in result.items() if k!='members'}))
|