Prepare single-process private v4 receiver and preserve execution ruling

This commit is contained in:
Frank14f committed 2026-11-16 12:14:06 +08:00
1 parent 99a70bbd99
commit 514235f5d8
4 files changed
+137

No files matched your search

@@ -0,0 +1,13 @@
{
"status": "PASS",
"checks": [
"full size and hashes",
"one process resumes retained exact offset",
"invalid range stops without append",
"MD5 mismatch preserves partial and stops",
"private URL and request exception text excluded",
"duplicate receiver start refused"
],
"cpu_seconds": 0.011480000000000018,
"scientific_data_read": false
}
@@ -105,3 +105,9 @@ Claude1794759849将模型共享CPU帽提高至216000秒,准备原144000秒不
准备624完整终态614PASS/10wall_limit,未触预算未派发;原614成功/10排除先冻结后拟合。原systemd模型服务于05:19:20完成SCORED并exit0。06:37独立只读核10fit、predict/score均PASS,全部10checkpoint/4602预测数组哈希及冻结关联一致;无重算。三分母均先逐split求统计再平均,主RMS分母平均中位naive0.088539745292、GBRT0.068013417769、FNN0.076701548477;只能描述本配置,不能替代原v4或家族独立泛化结论。
研究目标与前人方法边界、10大件超时覆盖偏差、补充离散化、t_set尺度/naive比较限制及FNN单输出/早停、随机几何近重复风险并列记录于docs/gre24-v5-reproduction-results-2026-11-16.md。CPU136620.981045/216000秒,GPU876.330852390/7200秒;余量不扩张范围。此次阶段决定是固定结果与完整证据,交Claude主审、Mold/Grok仅需定点复核;尚不冒称主审已完成。不再拟合/预测/评分,不自动进入v4计算、R4或新算法,独立O1类型/连通仍未由此检验。
## Gre24 v4 已有裁量接续(2026-11-16)
已实读Claude1794801454/1794801485确认沿1794782740原预裁量执行v4:唯一OSS拉取、Horsea复核大小和官方MD5通过后,以v5相同固定科学/模型代码做315测试/314训练×5,预先固定种子;不做spatial-moment加速,每件1800秒/4GiB,新独立60workerCPUh/2GPUh,systemd脱离会话执行。v5原终态不重跑,报告同三分母、<0.1比例、配对中位数与版本敏感性。此项已有执行授权,不再重复用户go。
Grok1794801467唯一对象完成已实读,Horsea尚无文件/传输。Claude1794802279纠正本机已有OSS访问的假设,指定私有Drive短期只读GET交付;等待file ID期间仅做接收准备和种子冻结,没有启动科学。当前实际状态见docs/gre24-v4-reception-and-preflight-2026-11-16.md;不要将上传完成等同于Horsea校验或科学已启动。
@@ -15,3 +15,5 @@ Horsea 本轮实查没有 v4 文件、相关传输或 Gre24 科学进程,可
原固定评分代码内有 v5 专用历史限制文字;v4报告须明确实际数据身份并注明这些原模板文字不适用,不能据其误称v4使用了v5数据。科学及评分代码保持原样。
本记录阶段尚未启动下载或科学队列。下一步是在接收入口可用后进行唯一拉取,Horsea大小和MD5均通过后,依据629件实际清单冻结合同与划分并启动既有批准的流程。v5完整终态及证据不变。
后续实际读到Claude1794802279.252979纠正“Horsea已有OSS访问”的假设:由Grok为唯一对象生成最长6小时的只读GET链接,放入私有Drive文件,仅共享file ID和到期时间。Melt读取后单进程断点续传,本地核大小/MD5;下载完成通知Grok清理该临时Drive文件。接收程序 `tools/receive_gre24_v4.py` 已准备,六项合成检查通过(完整校验、同进程按实际偏移续传、错误Range拒绝、MD5不符保留、日志不含链接、拒绝重复启动),合成CPU0.011480秒,无科学数据读取。私有Drive文件ID尚未收到,接收程序未运行。
+116
View File
@@ -0,0 +1,116 @@
"""One resumable private-OSS receiver; no scientific work and no URL logging."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import time
from urllib.parse import urlsplit
import requests
EXPECTED_BYTES = 35925187752
EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b'
HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com'
def save(path, value):
temporary = path.with_suffix('.tmp')
temporary.write_text(json.dumps(value, indent=2) + '\n')
temporary.replace(path)
def receive(root, private_url_file):
root.mkdir(parents=True, exist_ok=True)
with (root / 'receive.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'receive-started.json').exists():
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
url = private_url_file.read_text().strip()
parsed = urlsplit(url)
if parsed.scheme != 'https' or parsed.hostname != HOST or parsed.path != '/greif-v4/data.zip':
raise ValueError('Unexpected private object identity')
target = root / 'data.zip'
partial = root / 'data.zip.partial'
if target.exists():
raise FileExistsError('Destination already exists')
save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(),
source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES,
expected_md5=EXPECTED_MD5, private_url_not_recorded=True))
start = time.monotonic()
try:
for attempt in range(1, 7):
offset = partial.stat().st_size if partial.exists() else 0
if offset == EXPECTED_BYTES:
break
if offset > EXPECTED_BYTES:
raise ValueError('Oversized partial file')
headers = {'Accept-Encoding': 'identity'}
if offset:
headers['Range'] = f'bytes={offset}-'
try:
with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response:
expected_status = 206 if offset else 200
if response.status_code != expected_status:
raise ValueError(f'Unexpected HTTP status {response.status_code}')
if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset:
raise ValueError('Unexpected response length')
if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}':
raise ValueError('Invalid resume range')
last = 0
with partial.open('ab') as output:
for chunk in response.iter_content(chunk_size=2**20):
if not chunk:
continue
if offset + len(chunk) > EXPECTED_BYTES:
raise ValueError('Response exceeds expected size')
output.write(chunk)
offset += len(chunk)
if time.monotonic() - last >= 5:
output.flush()
save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(),
bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt,
elapsed_seconds=time.monotonic()-start, unix=time.time()))
last = time.monotonic()
output.flush()
os.fsync(output.fileno())
except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error:
with (root / 'transport-events.jsonl').open('a') as log:
log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__,
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n')
if attempt == 6:
raise RuntimeError('Transport attempts exhausted') from None
time.sleep(5)
if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES:
raise ValueError('Incomplete object')
save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time()))
md5 = hashlib.md5()
sha256 = hashlib.sha256()
with partial.open('rb') as source:
for block in iter(lambda: source.read(8*2**20), b''):
md5.update(block)
sha256.update(block)
if md5.hexdigest() != EXPECTED_MD5:
save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5))
raise ValueError('MD5 mismatch; original partial retained')
partial.rename(target)
stat = target.stat()
save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()),
bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(),
sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start,
source='private OSS GET delivered through Drive; URL excluded', science_started=False))
save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False))
return 0
except Exception as error:
# Error text can contain a signed URL. Only record the exception class.
save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__,
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False))
return 1
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('private_url_file', type=Path)
args = parser.parse_args()
raise SystemExit(receive(args.root, args.private_url_file))