Prepare single-process private v4 receiver and preserve execution ruling
This commit is contained in:
1 parent
99a70bbd99
commit
514235f5d8
4 files changed
+137
No files matched your search
@@ -0,0 +1,116 @@
|
||||
"""One resumable private-OSS receiver; no scientific work and no URL logging."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import requests
|
||||
|
||||
EXPECTED_BYTES = 35925187752
|
||||
EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b'
|
||||
HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com'
|
||||
|
||||
|
||||
def save(path, value):
|
||||
temporary = path.with_suffix('.tmp')
|
||||
temporary.write_text(json.dumps(value, indent=2) + '\n')
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def receive(root, private_url_file):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
with (root / 'receive.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'receive-started.json').exists():
|
||||
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
|
||||
url = private_url_file.read_text().strip()
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != 'https' or parsed.hostname != HOST or parsed.path != '/greif-v4/data.zip':
|
||||
raise ValueError('Unexpected private object identity')
|
||||
target = root / 'data.zip'
|
||||
partial = root / 'data.zip.partial'
|
||||
if target.exists():
|
||||
raise FileExistsError('Destination already exists')
|
||||
save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(),
|
||||
source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES,
|
||||
expected_md5=EXPECTED_MD5, private_url_not_recorded=True))
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for attempt in range(1, 7):
|
||||
offset = partial.stat().st_size if partial.exists() else 0
|
||||
if offset == EXPECTED_BYTES:
|
||||
break
|
||||
if offset > EXPECTED_BYTES:
|
||||
raise ValueError('Oversized partial file')
|
||||
headers = {'Accept-Encoding': 'identity'}
|
||||
if offset:
|
||||
headers['Range'] = f'bytes={offset}-'
|
||||
try:
|
||||
with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response:
|
||||
expected_status = 206 if offset else 200
|
||||
if response.status_code != expected_status:
|
||||
raise ValueError(f'Unexpected HTTP status {response.status_code}')
|
||||
if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset:
|
||||
raise ValueError('Unexpected response length')
|
||||
if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}':
|
||||
raise ValueError('Invalid resume range')
|
||||
last = 0
|
||||
with partial.open('ab') as output:
|
||||
for chunk in response.iter_content(chunk_size=2**20):
|
||||
if not chunk:
|
||||
continue
|
||||
if offset + len(chunk) > EXPECTED_BYTES:
|
||||
raise ValueError('Response exceeds expected size')
|
||||
output.write(chunk)
|
||||
offset += len(chunk)
|
||||
if time.monotonic() - last >= 5:
|
||||
output.flush()
|
||||
save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(),
|
||||
bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt,
|
||||
elapsed_seconds=time.monotonic()-start, unix=time.time()))
|
||||
last = time.monotonic()
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error:
|
||||
with (root / 'transport-events.jsonl').open('a') as log:
|
||||
log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__,
|
||||
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n')
|
||||
if attempt == 6:
|
||||
raise RuntimeError('Transport attempts exhausted') from None
|
||||
time.sleep(5)
|
||||
if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES:
|
||||
raise ValueError('Incomplete object')
|
||||
save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time()))
|
||||
md5 = hashlib.md5()
|
||||
sha256 = hashlib.sha256()
|
||||
with partial.open('rb') as source:
|
||||
for block in iter(lambda: source.read(8*2**20), b''):
|
||||
md5.update(block)
|
||||
sha256.update(block)
|
||||
if md5.hexdigest() != EXPECTED_MD5:
|
||||
save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5))
|
||||
raise ValueError('MD5 mismatch; original partial retained')
|
||||
partial.rename(target)
|
||||
stat = target.stat()
|
||||
save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()),
|
||||
bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(),
|
||||
sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start,
|
||||
source='private OSS GET delivered through Drive; URL excluded', science_started=False))
|
||||
save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False))
|
||||
return 0
|
||||
except Exception as error:
|
||||
# Error text can contain a signed URL. Only record the exception class.
|
||||
save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__,
|
||||
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('private_url_file', type=Path)
|
||||
args = parser.parse_args()
|
||||
raise SystemExit(receive(args.root, args.private_url_file))
|
||||
Reference in new issue
Block a user