"""One resumable private-OSS receiver; no scientific work and no URL logging.""" import argparse import fcntl import hashlib import json import os from pathlib import Path import time from urllib.parse import urlsplit, unquote import requests EXPECTED_BYTES = 35925187752 EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b' HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com' def save(path, value): temporary = path.with_suffix('.tmp') temporary.write_text(json.dumps(value, indent=2) + '\n') temporary.replace(path) def receive(root, private_url_file): root.mkdir(parents=True, exist_ok=True) with (root / 'receive.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root / 'receive-started.json').exists(): raise RuntimeError('Receiver already attempted; inspect receipts before continuation') url = private_url_file.read_text().strip() parsed = urlsplit(url) if parsed.scheme != 'https' or parsed.hostname != HOST or unquote(parsed.path) != '/greif-v4/data.zip': raise ValueError('Unexpected private object identity') target = root / 'data.zip' partial = root / 'data.zip.partial' if target.exists(): raise FileExistsError('Destination already exists') save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(), source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES, expected_md5=EXPECTED_MD5, private_url_not_recorded=True)) start = time.monotonic() try: for attempt in range(1, 7): offset = partial.stat().st_size if partial.exists() else 0 if offset == EXPECTED_BYTES: break if offset > EXPECTED_BYTES: raise ValueError('Oversized partial file') headers = {'Accept-Encoding': 'identity'} if offset: headers['Range'] = f'bytes={offset}-' try: with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response: expected_status = 206 if offset else 200 if response.status_code != expected_status: raise ValueError(f'Unexpected HTTP status {response.status_code}') if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset: raise ValueError('Unexpected response length') if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}': raise ValueError('Invalid resume range') last = 0 with partial.open('ab') as output: for chunk in response.iter_content(chunk_size=2**20): if not chunk: continue if offset + len(chunk) > EXPECTED_BYTES: raise ValueError('Response exceeds expected size') output.write(chunk) offset += len(chunk) if time.monotonic() - last >= 5: output.flush() save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(), bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt, elapsed_seconds=time.monotonic()-start, unix=time.time())) last = time.monotonic() output.flush() os.fsync(output.fileno()) except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error: with (root / 'transport-events.jsonl').open('a') as log: log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__, retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n') if attempt == 6: raise RuntimeError('Transport attempts exhausted') from None time.sleep(5) if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES: raise ValueError('Incomplete object') save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time())) md5 = hashlib.md5() sha256 = hashlib.sha256() with partial.open('rb') as source: for block in iter(lambda: source.read(8*2**20), b''): md5.update(block) sha256.update(block) if md5.hexdigest() != EXPECTED_MD5: save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5)) raise ValueError('MD5 mismatch; original partial retained') partial.rename(target) stat = target.stat() save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()), bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(), sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start, source='private OSS GET delivered through Drive; URL excluded', science_started=False)) save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False)) return 0 except Exception as error: # Error text can contain a signed URL. Only record the exception class. save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__, retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False)) return 1 if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('root', type=Path) parser.add_argument('private_url_file', type=Path) args = parser.parse_args() raise SystemExit(receive(args.root, args.private_url_file))