Files
MeltMind/tools/receive_gre24_v4.py

117 lines
6.4 KiB
Python

"""One resumable private-OSS receiver; no scientific work and no URL logging."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import time
from urllib.parse import urlsplit, unquote
import requests
EXPECTED_BYTES = 35925187752
EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b'
HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com'
def save(path, value):
temporary = path.with_suffix('.tmp')
temporary.write_text(json.dumps(value, indent=2) + '\n')
temporary.replace(path)
def receive(root, private_url_file):
root.mkdir(parents=True, exist_ok=True)
with (root / 'receive.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'receive-started.json').exists():
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
url = private_url_file.read_text().strip()
parsed = urlsplit(url)
if parsed.scheme != 'https' or parsed.hostname != HOST or unquote(parsed.path) != '/greif-v4/data.zip':
raise ValueError('Unexpected private object identity')
target = root / 'data.zip'
partial = root / 'data.zip.partial'
if target.exists():
raise FileExistsError('Destination already exists')
save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(),
source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES,
expected_md5=EXPECTED_MD5, private_url_not_recorded=True))
start = time.monotonic()
try:
for attempt in range(1, 7):
offset = partial.stat().st_size if partial.exists() else 0
if offset == EXPECTED_BYTES:
break
if offset > EXPECTED_BYTES:
raise ValueError('Oversized partial file')
headers = {'Accept-Encoding': 'identity'}
if offset:
headers['Range'] = f'bytes={offset}-'
try:
with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response:
expected_status = 206 if offset else 200
if response.status_code != expected_status:
raise ValueError(f'Unexpected HTTP status {response.status_code}')
if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset:
raise ValueError('Unexpected response length')
if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}':
raise ValueError('Invalid resume range')
last = 0
with partial.open('ab') as output:
for chunk in response.iter_content(chunk_size=2**20):
if not chunk:
continue
if offset + len(chunk) > EXPECTED_BYTES:
raise ValueError('Response exceeds expected size')
output.write(chunk)
offset += len(chunk)
if time.monotonic() - last >= 5:
output.flush()
save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(),
bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt,
elapsed_seconds=time.monotonic()-start, unix=time.time()))
last = time.monotonic()
output.flush()
os.fsync(output.fileno())
except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error:
with (root / 'transport-events.jsonl').open('a') as log:
log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__,
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n')
if attempt == 6:
raise RuntimeError('Transport attempts exhausted') from None
time.sleep(5)
if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES:
raise ValueError('Incomplete object')
save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time()))
md5 = hashlib.md5()
sha256 = hashlib.sha256()
with partial.open('rb') as source:
for block in iter(lambda: source.read(8*2**20), b''):
md5.update(block)
sha256.update(block)
if md5.hexdigest() != EXPECTED_MD5:
save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5))
raise ValueError('MD5 mismatch; original partial retained')
partial.rename(target)
stat = target.stat()
save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()),
bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(),
sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start,
source='private OSS GET delivered through Drive; URL excluded', science_started=False))
save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False))
return 0
except Exception as error:
# Error text can contain a signed URL. Only record the exception class.
save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__,
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False))
return 1
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('private_url_file', type=Path)
args = parser.parse_args()
raise SystemExit(receive(args.root, args.private_url_file))