117 lines
6.4 KiB
Python
117 lines
6.4 KiB
Python
"""One resumable private-OSS receiver; no scientific work and no URL logging."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
from urllib.parse import urlsplit, unquote
|
|
import requests
|
|
|
|
EXPECTED_BYTES = 35925187752
|
|
EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b'
|
|
HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com'
|
|
|
|
|
|
def save(path, value):
|
|
temporary = path.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(value, indent=2) + '\n')
|
|
temporary.replace(path)
|
|
|
|
|
|
def receive(root, private_url_file):
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
with (root / 'receive.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root / 'receive-started.json').exists():
|
|
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
|
|
url = private_url_file.read_text().strip()
|
|
parsed = urlsplit(url)
|
|
if parsed.scheme != 'https' or parsed.hostname != HOST or unquote(parsed.path) != '/greif-v4/data.zip':
|
|
raise ValueError('Unexpected private object identity')
|
|
target = root / 'data.zip'
|
|
partial = root / 'data.zip.partial'
|
|
if target.exists():
|
|
raise FileExistsError('Destination already exists')
|
|
save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(),
|
|
source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES,
|
|
expected_md5=EXPECTED_MD5, private_url_not_recorded=True))
|
|
start = time.monotonic()
|
|
try:
|
|
for attempt in range(1, 7):
|
|
offset = partial.stat().st_size if partial.exists() else 0
|
|
if offset == EXPECTED_BYTES:
|
|
break
|
|
if offset > EXPECTED_BYTES:
|
|
raise ValueError('Oversized partial file')
|
|
headers = {'Accept-Encoding': 'identity'}
|
|
if offset:
|
|
headers['Range'] = f'bytes={offset}-'
|
|
try:
|
|
with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response:
|
|
expected_status = 206 if offset else 200
|
|
if response.status_code != expected_status:
|
|
raise ValueError(f'Unexpected HTTP status {response.status_code}')
|
|
if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset:
|
|
raise ValueError('Unexpected response length')
|
|
if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}':
|
|
raise ValueError('Invalid resume range')
|
|
last = 0
|
|
with partial.open('ab') as output:
|
|
for chunk in response.iter_content(chunk_size=2**20):
|
|
if not chunk:
|
|
continue
|
|
if offset + len(chunk) > EXPECTED_BYTES:
|
|
raise ValueError('Response exceeds expected size')
|
|
output.write(chunk)
|
|
offset += len(chunk)
|
|
if time.monotonic() - last >= 5:
|
|
output.flush()
|
|
save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(),
|
|
bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt,
|
|
elapsed_seconds=time.monotonic()-start, unix=time.time()))
|
|
last = time.monotonic()
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error:
|
|
with (root / 'transport-events.jsonl').open('a') as log:
|
|
log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__,
|
|
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n')
|
|
if attempt == 6:
|
|
raise RuntimeError('Transport attempts exhausted') from None
|
|
time.sleep(5)
|
|
if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES:
|
|
raise ValueError('Incomplete object')
|
|
save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time()))
|
|
md5 = hashlib.md5()
|
|
sha256 = hashlib.sha256()
|
|
with partial.open('rb') as source:
|
|
for block in iter(lambda: source.read(8*2**20), b''):
|
|
md5.update(block)
|
|
sha256.update(block)
|
|
if md5.hexdigest() != EXPECTED_MD5:
|
|
save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5))
|
|
raise ValueError('MD5 mismatch; original partial retained')
|
|
partial.rename(target)
|
|
stat = target.stat()
|
|
save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()),
|
|
bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(),
|
|
sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start,
|
|
source='private OSS GET delivered through Drive; URL excluded', science_started=False))
|
|
save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False))
|
|
return 0
|
|
except Exception as error:
|
|
# Error text can contain a signed URL. Only record the exception class.
|
|
save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__,
|
|
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False))
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('root', type=Path)
|
|
parser.add_argument('private_url_file', type=Path)
|
|
args = parser.parse_args()
|
|
raise SystemExit(receive(args.root, args.private_url_file))
|