Files
MeltMind/artifacts/archive-runs-2026-11-17/inventory-builder.py
T

93 lines
7.3 KiB
Python

import csv,hashlib,json,os,re,subprocess,time
from pathlib import Path
from collections import defaultdict
root=Path('/home/frank14f/optane/MeltMind'); repo=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
out=root/'artifacts/archive-runs-2026-11-17';out.mkdir(exist_ok=True)
ref='036290bb23ffdc4adf85fb12edc2cd7c3485a4e8'
def save(name,x): (out/name).write_text(json.dumps(x,ensure_ascii=False,indent=2)+'\n')
def write_table(name,fields,rows):
with (out/name).open('w',newline='') as f:
w=csv.writer(f,delimiter='\t',lineterminator='\n');w.writerow(fields);w.writerows(rows)
def category(p):
if p.startswith('docs/'): return 'report_contract_decision'
if p.startswith('references/'): return 'retrieved_source_historical'
if p.startswith('tools/'): return 'implementation_or_test'
if re.search(r'contract|freeze|source|manifest',p): return 'identity_contract_freeze'
if re.search(r'failure|stop|incident|interruption',p): return 'failure_or_stop_evidence'
if re.search(r'guard|receipt|audit|ledger|complete|progress|started',p): return 'execution_evidence'
if re.search(r'prediction|scores|tables|contrasts',p): return 'prediction_or_score'
if re.search(r'\.npz$|\.npy$|\.joblib$|\.pt$|\.pth$|\.zip$',p): return 'raw_array_model_archive'
return 'other_retained'
raw=subprocess.check_output(['git','ls-tree','-rlz',ref],cwd=repo)
git={}
for entry in raw.split(b'\0'):
if not entry: continue
meta,p=entry.split(b'\t',1);mode,kind,oid,size=meta.decode().split();p=p.decode()
git[p]=(oid,int(size) if size!='-' else 0)
write_table('git-files.tsv',['path','git_blob_oid','bytes','category','baseline_commit'],[(p,*v,category(p),ref) for p,v in sorted(git.items())])
local={};excluded=[]
for top in ['docs','references','tools','artifacts']:
for current,dirs,files in os.walk(root/top,followlinks=False):
keep=[]
for d in dirs:
p=Path(current)/d;rel=p.relative_to(root).as_posix()
if d in {'.git','__pycache__','.pytest_cache','venv','node_modules'} or rel=='artifacts/archive-runs-2026-11-17': excluded.append(rel);continue
keep.append(d)
dirs[:]=keep
for name in files:
p=Path(current)/name; rel=p.relative_to(root).as_posix()
if p.is_symlink(): continue
s=p.stat();local[rel]=(s.st_size,s.st_mtime_ns)
for p in [root/'data.zip',root/'AGENTS.md']:
if p.exists():s=p.stat();local[p.relative_to(root).as_posix()]=(s.st_size,s.st_mtime_ns)
local_rows=[];only=[];mismatch=[]
for p,(size,mtime) in sorted(local.items()):
state='git_baseline_same_size_unhashed' if p in git and git[p][1]==size else 'local_only' if p not in git else 'git_baseline_size_differs'
row=(p,size,mtime,state,category(p));local_rows.append(row)
if p not in git:only.append(row)
elif size!=git[p][1]:mismatch.append(row)
fields=['path','bytes','mtime_ns','publication_observation','category']
write_table('local-files.tsv',fields,local_rows);write_table('local-only.tsv',fields,only);write_table('local-size-differences.tsv',fields,mismatch)
docs=[]
for p,(size,_) in sorted(local.items()):
if not (p.startswith(('docs/','references/','tools/')) and size<10_000_000):continue
f=root/p
if f.suffix not in {'.md','.txt','.py','.json','.csv','.sh'}:continue
data=f.read_bytes(); title=''
if f.suffix in {'.md','.txt'}:
title=next((x.strip('# ').strip() for x in data.decode(errors='replace').splitlines() if x.strip()),'')[:180]
docs.append((p,category(p),title,len(data),hashlib.sha256(data).hexdigest(),git.get(p,('',0))[0]))
write_table('documents.tsv',['path','category','title','bytes','local_sha256','baseline_git_blob_oid'],docs)
by_root=defaultdict(list)
for p in local:
if p.startswith('artifacts/') and len(Path(p).parts)>=3:by_root[Path(p).parts[1]].append(p)
interesting=re.compile(r'contract|freeze|complete|completion|ledger|audit|receipt|failure|verification|acceptance|cost|source|plan|terminal')
def scalar_evidence(obj,prefix='',depth=0):
ans={}
if not isinstance(obj,dict) or depth>3:return ans
for k,v in obj.items():
key=prefix+'.'+k if prefix else k
if isinstance(v,dict) and depth<3 and len(v)<80:ans.update(scalar_evidence(v,key,depth+1))
elif isinstance(v,(str,int,float,bool)) or v is None:
if re.search(r'status|cpu|gpu|wall|commit|sha256|account|stop_reason|implementation|source_pin',key,re.I) and len(str(v))<300:ans[key]=v
return dict(list(ans.items())[:100])
roots=[]
for name,paths in sorted(by_root.items()):
receipts=[]
for p in sorted(paths):
rel=Path(p).relative_to('artifacts/'+name)
if len(rel.parts)>3 or not interesting.search(rel.name) or not p.endswith('.json'):continue
data=(root/p).read_bytes()
if len(data)>5_000_000:continue
try: obj=json.loads(data)
except Exception: continue
receipts.append(dict(path=p,bytes=len(data),sha256=hashlib.sha256(data).hexdigest(),git_blob_oid=git.get(p,('',0))[0],observed_fields=scalar_evidence(obj)))
related=[]
family=next((s for s in ['s3b-a','s3a','s2','s1','r4bc','r4a','o1','o2','gre24','naive','log'] if name.startswith(s)),None)
for p,cat,title,size,h,g in docs:
if p.startswith('docs/') and family and Path(p).name.startswith(family): related.append(p)
roots.append(dict(root='artifacts/'+name,absolute_root=str(root/'artifacts'/name),archive_review_status='open: consult linked final report and review ruling; indexing alone is not acceptance',local_file_count=len(paths),local_bytes=sum(local[p][0] for p in paths),git_baseline_files=sum(p in git for p in paths),local_only_files=sum(p not in git for p in paths),related_documents=related,identity_terminal_cost_receipts=receipts,receipt_selection='JSON names matching contract/freeze/complete/ledger/audit/receipt/failure/verification/acceptance/cost/source/plan/terminal; at most 3 levels inside root, <=5MB; deeper per-job files remain in full path inventories; observed progress fields are historical, not authoritative current status'))
save('run-roots.json',roots)
save('inventory-summary.json',dict(status='METADATA_INDEX_COMPLETE_PENDING_INDEPENDENT_COMPLETENESS_REVIEW',baseline_commit=ref,created_unix=time.time(),owner='MeltAstra',instruction_slack_ts='1794880293.742719',git_files=len(git),git_bytes=sum(v[1] for v in git.values()),local_files=len(local),local_bytes=sum(v[0] for v in local.values()),local_only_files=len(only),size_different_files=len(mismatch),run_roots=len(roots),documents_and_sources_hashed=len(docs),receipt_files_hashed=sum(len(x['identity_terminal_cost_receipts']) for x in roots),exclusions=sorted(set(excluded)),limitations=['Only docs/references/tools/artifacts plus root data.zip and AGENTS.md inventoried locally; no whole-home scan','Third-party environments, caches, symlinks and this new inventory itself excluded','Same size is not content equality; Git blob IDs identify committed bytes, local SHA256 supplied only for indexed small documents and selected JSON receipts','No experiment, decoding, prediction, scoring, bootstrap, model fit or scientific test executed','No historical raw artifacts deleted, altered, moved or permission-changed','No total project cost reconstructed: overlapping cumulative ledgers must not be summed','Some primary reports contain historical running snapshots; terminal reports and original receipts govern','Archive metadata review status open is not authorization or a pending scientific queue']))
print(json.dumps({k:v for k,v in json.loads((out/'inventory-summary.json').read_text()).items() if k not in ['exclusions','limitations']},ensure_ascii=False,indent=2))