Prepare single-process private v4 receiver and preserve execution ruling
This commit is contained in:
1 parent
99a70bbd99
commit
514235f5d8
4 files changed
+137
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"checks": [
|
||||
"full size and hashes",
|
||||
"one process resumes retained exact offset",
|
||||
"invalid range stops without append",
|
||||
"MD5 mismatch preserves partial and stops",
|
||||
"private URL and request exception text excluded",
|
||||
"duplicate receiver start refused"
|
||||
],
|
||||
"cpu_seconds": 0.011480000000000018,
|
||||
"scientific_data_read": false
|
||||
}
|
||||
@@ -105,3 +105,9 @@ Claude1794759849将模型共享CPU帽提高至216000秒,准备原144000秒不
|
||||
准备624完整终态614PASS/10wall_limit,未触预算未派发;原614成功/10排除先冻结后拟合。原systemd模型服务于05:19:20完成SCORED并exit0。06:37独立只读核10fit、predict/score均PASS,全部10checkpoint/4602预测数组哈希及冻结关联一致;无重算。三分母均先逐split求统计再平均,主RMS分母平均中位naive0.088539745292、GBRT0.068013417769、FNN0.076701548477;只能描述本配置,不能替代原v4或家族独立泛化结论。
|
||||
|
||||
研究目标与前人方法边界、10大件超时覆盖偏差、补充离散化、t_set尺度/naive比较限制及FNN单输出/早停、随机几何近重复风险并列记录于docs/gre24-v5-reproduction-results-2026-11-16.md。CPU136620.981045/216000秒,GPU876.330852390/7200秒;余量不扩张范围。此次阶段决定是固定结果与完整证据,交Claude主审、Mold/Grok仅需定点复核;尚不冒称主审已完成。不再拟合/预测/评分,不自动进入v4计算、R4或新算法,独立O1类型/连通仍未由此检验。
|
||||
|
||||
## Gre24 v4 已有裁量接续(2026-11-16)
|
||||
|
||||
已实读Claude1794801454/1794801485确认沿1794782740原预裁量执行v4:唯一OSS拉取、Horsea复核大小和官方MD5通过后,以v5相同固定科学/模型代码做315测试/314训练×5,预先固定种子;不做spatial-moment加速,每件1800秒/4GiB,新独立60workerCPUh/2GPUh,systemd脱离会话执行。v5原终态不重跑,报告同三分母、<0.1比例、配对中位数与版本敏感性。此项已有执行授权,不再重复用户go。
|
||||
|
||||
Grok1794801467唯一对象完成已实读,Horsea尚无文件/传输。Claude1794802279纠正本机已有OSS访问的假设,指定私有Drive短期只读GET交付;等待file ID期间仅做接收准备和种子冻结,没有启动科学。当前实际状态见docs/gre24-v4-reception-and-preflight-2026-11-16.md;不要将上传完成等同于Horsea校验或科学已启动。
|
||||
@@ -15,3 +15,5 @@ Horsea 本轮实查没有 v4 文件、相关传输或 Gre24 科学进程,可
|
||||
原固定评分代码内有 v5 专用历史限制文字;v4报告须明确实际数据身份并注明这些原模板文字不适用,不能据其误称v4使用了v5数据。科学及评分代码保持原样。
|
||||
|
||||
本记录阶段尚未启动下载或科学队列。下一步是在接收入口可用后进行唯一拉取,Horsea大小和MD5均通过后,依据629件实际清单冻结合同与划分并启动既有批准的流程。v5完整终态及证据不变。
|
||||
|
||||
后续实际读到Claude1794802279.252979纠正“Horsea已有OSS访问”的假设:由Grok为唯一对象生成最长6小时的只读GET链接,放入私有Drive文件,仅共享file ID和到期时间。Melt读取后单进程断点续传,本地核大小/MD5;下载完成通知Grok清理该临时Drive文件。接收程序 `tools/receive_gre24_v4.py` 已准备,六项合成检查通过(完整校验、同进程按实际偏移续传、错误Range拒绝、MD5不符保留、日志不含链接、拒绝重复启动),合成CPU0.011480秒,无科学数据读取。私有Drive文件ID尚未收到,接收程序未运行。
|
||||
@@ -0,0 +1,116 @@
|
||||
"""One resumable private-OSS receiver; no scientific work and no URL logging."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import requests
|
||||
|
||||
EXPECTED_BYTES = 35925187752
|
||||
EXPECTED_MD5 = '4973f24e12a38c87c31cf129be1a3e3b'
|
||||
HOST = 'frank14f-private-hk.oss-cn-hongkong.aliyuncs.com'
|
||||
|
||||
|
||||
def save(path, value):
|
||||
temporary = path.with_suffix('.tmp')
|
||||
temporary.write_text(json.dumps(value, indent=2) + '\n')
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def receive(root, private_url_file):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
with (root / 'receive.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'receive-started.json').exists():
|
||||
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
|
||||
url = private_url_file.read_text().strip()
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != 'https' or parsed.hostname != HOST or parsed.path != '/greif-v4/data.zip':
|
||||
raise ValueError('Unexpected private object identity')
|
||||
target = root / 'data.zip'
|
||||
partial = root / 'data.zip.partial'
|
||||
if target.exists():
|
||||
raise FileExistsError('Destination already exists')
|
||||
save(root / 'receive-started.json', dict(pid=os.getpid(), unix=time.time(),
|
||||
source_host=HOST, source_key='greif-v4/data.zip', expected_bytes=EXPECTED_BYTES,
|
||||
expected_md5=EXPECTED_MD5, private_url_not_recorded=True))
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for attempt in range(1, 7):
|
||||
offset = partial.stat().st_size if partial.exists() else 0
|
||||
if offset == EXPECTED_BYTES:
|
||||
break
|
||||
if offset > EXPECTED_BYTES:
|
||||
raise ValueError('Oversized partial file')
|
||||
headers = {'Accept-Encoding': 'identity'}
|
||||
if offset:
|
||||
headers['Range'] = f'bytes={offset}-'
|
||||
try:
|
||||
with requests.get(url, headers=headers, stream=True, timeout=(30, 120), allow_redirects=False) as response:
|
||||
expected_status = 206 if offset else 200
|
||||
if response.status_code != expected_status:
|
||||
raise ValueError(f'Unexpected HTTP status {response.status_code}')
|
||||
if int(response.headers['Content-Length']) != EXPECTED_BYTES - offset:
|
||||
raise ValueError('Unexpected response length')
|
||||
if offset and response.headers.get('Content-Range') != f'bytes {offset}-{EXPECTED_BYTES-1}/{EXPECTED_BYTES}':
|
||||
raise ValueError('Invalid resume range')
|
||||
last = 0
|
||||
with partial.open('ab') as output:
|
||||
for chunk in response.iter_content(chunk_size=2**20):
|
||||
if not chunk:
|
||||
continue
|
||||
if offset + len(chunk) > EXPECTED_BYTES:
|
||||
raise ValueError('Response exceeds expected size')
|
||||
output.write(chunk)
|
||||
offset += len(chunk)
|
||||
if time.monotonic() - last >= 5:
|
||||
output.flush()
|
||||
save(root / 'receive-progress.json', dict(status='DOWNLOADING', pid=os.getpid(),
|
||||
bytes=offset, expected_bytes=EXPECTED_BYTES, attempt=attempt,
|
||||
elapsed_seconds=time.monotonic()-start, unix=time.time()))
|
||||
last = time.monotonic()
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError) as error:
|
||||
with (root / 'transport-events.jsonl').open('a') as log:
|
||||
log.write(json.dumps(dict(attempt=attempt, error_type=type(error).__name__,
|
||||
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time()))+'\n')
|
||||
if attempt == 6:
|
||||
raise RuntimeError('Transport attempts exhausted') from None
|
||||
time.sleep(5)
|
||||
if not partial.exists() or partial.stat().st_size != EXPECTED_BYTES:
|
||||
raise ValueError('Incomplete object')
|
||||
save(root / 'receive-progress.json', dict(status='VERIFYING_MD5', pid=os.getpid(), bytes=EXPECTED_BYTES, unix=time.time()))
|
||||
md5 = hashlib.md5()
|
||||
sha256 = hashlib.sha256()
|
||||
with partial.open('rb') as source:
|
||||
for block in iter(lambda: source.read(8*2**20), b''):
|
||||
md5.update(block)
|
||||
sha256.update(block)
|
||||
if md5.hexdigest() != EXPECTED_MD5:
|
||||
save(root / 'hash-mismatch.json', dict(observed_md5=md5.hexdigest(), expected_md5=EXPECTED_MD5))
|
||||
raise ValueError('MD5 mismatch; original partial retained')
|
||||
partial.rename(target)
|
||||
stat = target.stat()
|
||||
save(root / 'archive-verification.json', dict(status='VERIFIED', archive=str(target.resolve()),
|
||||
bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, inode=stat.st_ino, md5=md5.hexdigest(),
|
||||
sha256=sha256.hexdigest(), unix=time.time(), elapsed_seconds=time.monotonic()-start,
|
||||
source='private OSS GET delivered through Drive; URL excluded', science_started=False))
|
||||
save(root / 'receive-complete.json', dict(status='VERIFIED', pid=os.getpid(), unix=time.time(), science_started=False))
|
||||
return 0
|
||||
except Exception as error:
|
||||
# Error text can contain a signed URL. Only record the exception class.
|
||||
save(root / 'receive-complete.json', dict(status='STOPPED', error_type=type(error).__name__,
|
||||
retained_bytes=partial.stat().st_size if partial.exists() else 0, unix=time.time(), science_started=False))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('private_url_file', type=Path)
|
||||
args = parser.parse_args()
|
||||
raise SystemExit(receive(args.root, args.private_url_file))
|
||||
Reference in new issue
Block a user