o
    6‹Tjà  ã                   @  s�   d Z ddlmZ ddlZddlZddlZddlmZ ej 	de
eeƒ ¡ jd d ƒ¡ ddlmZ ddlmZ ddd„ZedkrFeƒ  dS dS )aq  Embed the RAG chunks and build a FAISS index.

    python scripts/build_index.py --config config/config.yaml

Reads data/kb/chunks.jsonl (from prepare_data.py), embeds each chunk with the small embedder,
and writes:
  - data/kb/index.faiss       (inner-product index over L2-normalized vectors == cosine)
  - data/kb/index.meta.json   (parallel list of {text, source})
é    )ÚannotationsN)ÚPathé   Úsrc)ÚEmbedder)Úload_configÚreturnÚNonec            
      C  s  t  ¡ } | jddd� |  ¡ }t|jƒ}dd l}| d¡}|d }| ¡ s,t	|› d�ƒ‚dd	„ |j
d
d� ¡ D ƒ}|s?t	dƒ‚t|ƒ}| dd	„ |D ƒ¡}| |jd ¡}	|	 |¡ | |	t|d ƒ¡ |d jtj|dd�d
d� tdt|ƒ› d|jd › d|d › �ƒ d S )Nz--configzconfig/config.yaml)Údefaultr   zdata.kb_dirzchunks.jsonlz. not found. Run scripts/prepare_data.py first.c                 S  s   g | ]}|  ¡ rt |¡‘qS © )ÚstripÚjsonÚloads)Ú.0Úliner   r   úscripts/build_index.pyÚ
<listcomp>$   s    zmain.<locals>.<listcomp>zutf-8)ÚencodingzJNo chunks to index. Add KB documents to data/raw/ and re-run prepare_data.c                 S  s   g | ]}|d  ‘qS )Útextr   )r   Úmr   r   r   r   )   s    r   zindex.faisszindex.meta.jsonF)Zensure_asciizIndexed z chunks (dim=z) -> )ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsr   ZconfigÚfaissÚresolveÚexistsÚ
SystemExitZ	read_textÚ
splitlinesr   Zembed_documentsZIndexFlatIPÚshapeÚaddZwrite_indexÚstrZ
write_textr   ÚdumpsÚprintÚlen)
ZapÚargsZcfgr   Zkb_dirZchunks_pathÚmetaZembedderZvecsÚindexr   r   r   Úmain   s&   


,r(   Ú__main__)r   r	   )Ú__doc__Z
__future__r   r   r   ÚsysZpathlibr   ÚpathÚinsertr!   Ú__file__r   ÚparentsZagent.embeddingsr   Zagent.settingsr   r(   Ú__name__r   r   r   r   Ú<module>   s    	$

ÿ