o
    n�Tj  ã                   @  sÌ   d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ej
 dee	eƒ ¡ jd d ƒ¡ ddlZddlmZ ddlmZ dd	lmZ G d
d„ dƒZddd„Zddd„Zedkrdeƒ  dS dS )a}  Evaluate quality + energy of the agent on the held-out eval set.

    python scripts/evaluate.py --config config/config.yaml     # vLLM server must be running

Quality (per eval example {question, answer, source}):
  - answer_similarity : cosine sim between generated and reference answer (small embedder)
  - has_citation      : did the answer cite a source?
  - source_match      : did it cite the expected source?
Energy:
  - samples GPU power via NVML in a background thread -> integrates to Wh per query
  - reports cache hit-rate (cached answers cost ~0 GPU)

Writes a JSON report to reports/eval_report.json and prints a summary.
é    )ÚannotationsN)ÚPathé   Úsrc)ÚEmbedder)ÚAgent)Úload_configc                   @  sB   e Zd ZdZdddd„Zdd„ Zd	d
„ Zdd„ Zeddd„ƒZ	dS )ÚPowerSamplerzWBackground NVML power sampler. Integrates watts over time -> Wh. No-op if NVML missing.çš™™™™™É?ÚintervalÚfloatc              
   C  sˆ   || _ d| _t ¡ | _d | _d | _zdd l}| ¡  || _	| 
d¡| _W d S  tyC } zd | _	td|› d�ƒ W Y d }~d S d }~ww )Ng        r   z9  (energy: NVML unavailable, skipping power measurement: ú))r   Ú
_energy_whÚ	threadingZEventÚ_stopÚ_threadÚ_handleÚpynvmlZnvmlInitÚ_pynvmlZnvmlDeviceGetHandleByIndexÚ	ExceptionÚprint)Úselfr   r   Úe© r   úscripts/evaluate.pyÚ__init__$   s   
€þzPowerSampler.__init__c                 C  s€   t   ¡ }| j ¡ s>t  | j¡ t   ¡ }z| j | j¡}|  j|d || d  7  _W n	 t	y4   Y nw |}| j ¡ r	d S d S )Ng     @�@g      ¬@)
Útimer   Zis_setÚsleepr   r   ZnvmlDeviceGetPowerUsager   r   r   )r   ZlastZnowZmwr   r   r   Ú_run4   s   
"ÿøzPowerSampler._runc                 C  s*   | j d urtj| jdd�| _| j ¡  | S )NT)ÚtargetZdaemon)r   r   ZThreadr   r   Ústart©r   r   r   r   Ú	__enter__@   s   

zPowerSampler.__enter__c                 G  s"   | j  ¡  | jr| j ¡  d S d S ©N)r   Úsetr   Újoin)r   Úexcr   r   r   Ú__exit__F   s   
ÿzPowerSampler.__exit__Úreturnc                 C  s   | j S r#   )r   r!   r   r   r   Ú	energy_whK   s   zPowerSampler.energy_whN)r
   )r   r   )r(   r   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r"   r'   Úpropertyr)   r   r   r   r   r	   !   s    r	   Úaú
np.ndarrayÚbr(   r   c                 C  s   t t | |¡ƒS r#   )r   ÚnpÚdot)r/   r1   r   r   r   ÚcosineP   s   r4   ÚNonec                    s†  t  ¡ } | jddd� | jdtddd� |  ¡ }t|jƒ}| d¡}| ¡ s-t	|› d	�ƒ‚d
d„ |j
dd� ¡ D ƒ}|jrD|d |j… }t|dd�}t|ƒ}g }d}tƒ �p}	t ¡ }
|D ][}| |d ¡}| |j¡}| |d ¡}t||ƒ}| d¡p}d ¡ ‰ tˆ ƒo�t‡ fdd„|jD ƒƒ}|t|jƒ7 }| |d t|dƒt|jƒ||j|j|j|j|j|j |j!dœ¡ q\t ¡ |
 }W d   ƒ n1 sÈw   Y  t"|ƒ}t#dd„ |D ƒƒ}||rìtt$t% &dd„ |D ƒ¡ƒdƒnd|rýtt#dd„ |D ƒƒ| dƒnd|�rtt#dd„ |D ƒƒ| dƒnd|�r!tt#dd„ |D ƒƒ| dƒnd|�r,t|| dƒnd|�r>tt#dd„ |D ƒƒ| dƒnd|�rPtt#dd„ |D ƒƒ| dƒnd|�r[t|| dƒnd|t#d d„ |D ƒƒt|d!ƒt|	j'd"ƒ|�ryt|	j'| d"ƒndd#œ}t(t)ƒ ¡ j*d d$ d% }|j+j,ddd&� |j-t.j/||d'œd!d(�dd� t0d)ƒ | 1¡ D ]\}}t0d*|d+›d,|› �ƒ �q©t0d-|› �ƒ d S ).Nz--configzconfig/config.yaml)Údefaultz--limitr   z"evaluate only the first N examples)Útyper6   Úhelpzdata.sft_evalz. not found. Run scripts/prepare_data.py first.c                 S  s   g | ]}|  ¡ rt |¡‘qS r   )ÚstripÚjsonÚloads)Ú.0Úlr   r   r   Ú
<listcomp>^   s    zmain.<locals>.<listcomp>zutf-8)ÚencodingT)Zlazy_llmÚquestionÚanswerÚsourceÚ c                 3  s    � | ]}ˆ |v p|ˆ v V  qd S r#   r   )r<   Úc©r   r   r   Ú	<genexpr>o   ó   € zmain.<locals>.<genexpr>é   )r@   Úanswer_similarityÚhas_citationÚsource_matchÚgroundedÚcachedÚrouteÚprompt_tokensÚcompletion_tokensÚtotal_tokensÚguardrail_reasonc                 s  ó   � | ]}|d  V  qdS )rQ   Nr   ©r<   Úrr   r   r   rF   ƒ   ó   € c                 S  s   g | ]}|d  ‘qS )rI   r   rT   r   r   r   r>   †   s    c                 s  rS   )rJ   Nr   rT   r   r   r   rF   ‡   rV   c                 s  rS   )rK   Nr   rT   r   r   r   rF   ˆ   rV   c                 s  rS   )rL   Nr   rT   r   r   r   rF   ‰   rV   c                 s  rS   )rO   Nr   rT   r   r   r   rF   Œ   rV   r   c                 s  rS   )rP   Nr   rT   r   r   r   rF   �   rV   c                 s  s    � | ]}|d  dv rdV  qdS )rN   )ÚcacheZrouterr   Nr   rT   r   r   r   rF   �   rG   é   é   )ÚnZavg_answer_similarityZcitation_rateZsource_match_rateZgrounded_rateZcache_hit_rateZavg_prompt_tokensZavg_completion_tokensZavg_total_tokens_per_queryrQ   Zzero_token_queriesZwall_secondsZenergy_wh_totalZenergy_wh_per_queryZreportszeval_report.json)ÚparentsÚexist_ok)ÚsummaryÚrows)Úindentz
=== Evaluation summary ===z  Z22ú z
Full report -> )2ÚargparseÚArgumentParserÚadd_argumentÚintÚ
parse_argsr   ZconfigÚresolveÚexistsÚ
SystemExitZ	read_textÚ
splitlinesÚlimitr   r   r	   r   rA   Zembed_queryr4   Úgetr9   ÚboolÚanyZ	citationsrM   ÚappendÚroundrL   rN   rO   rP   rQ   rR   ÚlenÚsumr   r2   Zmeanr)   r   Ú__file__r[   ÚparentÚmkdirZ
write_textr:   Údumpsr   Úitems)ZapÚargsZcfgZ	eval_pathZexamplesZagentZembedderr^   Zn_cachedZsamplerZt0ÚexZrespZpred_vecZref_vecZsimrK   ZwallrZ   rQ   r]   Zreport_pathÚkÚvr   rE   r   ÚmainT   s‚   


 õÿç$"$$$$
ñr{   Ú__main__)r/   r0   r1   r0   r(   r   )r(   r5   )r-   Z
__future__r   ra   r:   Úsysr   r   Zpathlibr   ÚpathÚinsertÚstrrr   rf   r[   Znumpyr2   Zagent.embeddingsr   Zagent.pipeliner   Zagent.settingsr   r	   r4   r{   r*   r   r   r   r   Ú<module>   s&    $
/
L
ÿ