35 lines
2.2 KiB
Python
35 lines
2.2 KiB
Python
"""Epoch-boundary host observations only; no scheduling or causal decisions."""
|
|
import csv
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save
|
|
|
|
def epoch_host_observation(root,fold,epoch,gpu_uuid):
|
|
record=dict(unix=time.time(),fold=fold,completed_epoch=epoch,assigned_gpu_uuid=gpu_uuid,observation_only=True,sampling='After the coordinator reads the completed-epoch receipt; not continuous monitoring and not causal proof',gpu_processes=[],errors=[])
|
|
try:record['load_average']=list(os.getloadavg())
|
|
except OSError as exc:record['errors'].append(dict(stage='load_average',type=type(exc).__name__,reason=str(exc)))
|
|
own=set()
|
|
try:
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
args=p.info['cmdline'] or []
|
|
if str(root) in args and any(Path(a).name=='phase3_seed1_v2_worker.py' for a in args):own.add(p.pid)
|
|
except Exception as exc:record['errors'].append(dict(stage='own_worker_identity',type=type(exc).__name__,reason=str(exc)))
|
|
try:
|
|
result=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=5,check=True)
|
|
for row in csv.reader(io.StringIO(result.stdout)):
|
|
if not row:continue
|
|
if len(row)!=4:raise ValueError('Unexpected GPU process columns')
|
|
uuid,pid,name,memory=[s.strip() for s in row];pid=int(pid)
|
|
record['gpu_processes'].append(dict(gpu_uuid=uuid,pid=pid,process_name=name,used_memory_mib=memory,own_v2_worker=pid in own,on_assigned_gpu=uuid==gpu_uuid,foreign_on_assigned_gpu=uuid==gpu_uuid and pid not in own))
|
|
except Exception as exc:record['errors'].append(dict(stage='gpu_process_snapshot',type=type(exc).__name__,reason=str(exc)))
|
|
record['own_worker_pids']=sorted(own);record['status']='OBSERVED' if not record['errors'] else 'OBSERVATION_INCOMPLETE'
|
|
p=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json'
|
|
p.parent.mkdir(parents=True,exist_ok=True)
|
|
if p.exists():raise FileExistsError('Preserve original epoch observation')
|
|
save(p,record)
|
|
return record
|