"""Epoch-boundary host observations only; no scheduling or causal decisions.""" import csv import io import json import os from pathlib import Path import subprocess import time import psutil from phase2_h1_worker_v2 import save,sha def epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=None,incomplete_reason=None): record=dict(unix=time.time(),fold=fold,completed_epoch=epoch,assigned_gpu_uuid=gpu_uuid,observation_only=True,sampling='After the coordinator reads the completed-epoch receipt; not continuous monitoring and not causal proof',gpu_processes=[],errors=[]) if receipt is not None: receipt=Path(receipt) record.update(receipt_path=str(receipt),receipt_sha256=sha(receipt),receipt_mtime_unix=receipt.stat().st_mtime,receipt_age_seconds=max(0,record['unix']-receipt.stat().st_mtime)) if incomplete_reason is not None: record.update(status='OBSERVATION_INCOMPLETE',incomplete_reason=incomplete_reason,snapshot_started_unix=None,snapshot_finished_unix=None,own_worker_pids=[],historical_gpu_occupancy_observed=False) return write_observation(root,fold,epoch,record) record['snapshot_started_unix']=time.time() try:record['load_average']=list(os.getloadavg()) except OSError as exc:record['errors'].append(dict(stage='load_average',type=type(exc).__name__,reason=str(exc))) own=set() try: for p in psutil.process_iter(['pid','cmdline']): args=p.info['cmdline'] or [] if str(root) in args and any(Path(a).name=='phase3_seed1_v2_worker.py' for a in args):own.add(p.pid) except Exception as exc:record['errors'].append(dict(stage='own_worker_identity',type=type(exc).__name__,reason=str(exc))) try: result=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=5,check=True) for row in csv.reader(io.StringIO(result.stdout)): if not row:continue if len(row)!=4:raise ValueError('Unexpected GPU process columns') uuid,pid,name,memory=[s.strip() for s in row];pid=int(pid) record['gpu_processes'].append(dict(gpu_uuid=uuid,pid=pid,process_name=name,used_memory_mib=memory,own_v2_worker=pid in own,on_assigned_gpu=uuid==gpu_uuid,foreign_on_assigned_gpu=uuid==gpu_uuid and pid not in own)) except Exception as exc:record['errors'].append(dict(stage='gpu_process_snapshot',type=type(exc).__name__,reason=str(exc))) if not any(p['own_v2_worker'] and p['on_assigned_gpu'] for p in record['gpu_processes']): record['errors'].append(dict(stage='epoch_worker_visibility',reason='Assigned v2 worker not visible at snapshot; historical occupancy not observed')) record['own_worker_pids']=sorted(own);record['status']='OBSERVED' if not record['errors'] else 'OBSERVATION_INCOMPLETE' record['snapshot_finished_unix']=time.time() record['historical_gpu_occupancy_observed']=False return write_observation(root,fold,epoch,record) def write_observation(root,fold,epoch,record): p=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json' p.parent.mkdir(parents=True,exist_ok=True) if p.exists():raise FileExistsError('Preserve original epoch observation') save(p,record) return record def drain_epoch_observations(root,fold,gpu_uuid,covered,terminal=False,stopping=False): """Account for each completed receipt independently of the cost cursor. Terminal, stopping, batched or >5s-old receipts get explicit missing-history records without GPU queries. A fresh live singleton gets a current snapshot; its true times never claim coverage of earlier epoch occupancy. """ history=root/'jobs'/str(fold)/'worker/history' receipts=[] for p in sorted(history.glob('*.json')): row=json.loads(p.read_text());epoch=row['epoch'] if type(epoch) is not int or not 1<=epoch<=200 or p.stem!=f'{epoch:03d}':raise ValueError('Completed epoch receipt identity') if epoch not in covered:receipts.append((epoch,p)) backlog=len(receipts)>1;new_records=[] for epoch,p in receipts: output=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json' if output.exists(): record=json.loads(output.read_text()) if record['completed_epoch']!=epoch or record['fold']!=fold or record.get('receipt_sha256')!=sha(p):raise ValueError('Existing observation identity') else: reason=('TERMINAL_BEFORE_OBSERVATION' if terminal else 'GLOBAL_STOP_BEFORE_OBSERVATION' if stopping else 'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG' if backlog else 'DELAYED_RECEIPT_OVER_5_SECONDS' if time.time()-p.stat().st_mtime>5 else None) record=epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=p,incomplete_reason=reason) covered.add(epoch);new_records.append(record) return new_records