o
    ,‹TjÞ  ã                   @  sú   d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ej
 dee	eƒ ¡ jd d ƒ¡ ddlmZ ddlmZ e d	¡Zd+dd„Zd,dd„Zd-dd„Zd-dd„Zd.dd„Zd.dd„Zd/d"d#„Zd0d&d'„Zd1d(d)„Zed*kr{eƒ  dS dS )2a"  Prepare raw data into (a) SFT instruction JSONL and (b) a RAG chunk file.

    python scripts/prepare_data.py --config config/config.yaml

Reads everything in data.raw_dir:
  - *.jsonl with {question, answer, source?}  -> SFT pairs (train/eval split)
  - *.md / *.txt / *.csv                       -> chunked knowledge base -> data/kb/chunks.jsonl

SFT pairs are written in chat format ({"messages": [...]}) that TRL's SFTTrainer consumes, with
the system prompt from agent.prompt so fine-tuning reinforces the same behavioral contract we serve.
é    )ÚannotationsN)ÚPathé   Úsrc)ÚSYSTEM_PROMPT)Úload_configz^(#{1,6})\s+(.*)$ÚtextÚstrÚreturnc                 C  s    t  dd|  ¡ ¡ d¡}|pdS )Nz
[^a-z0-9]+ú-Úsection)ÚreÚsubÚlowerÚstrip)r   Ús© r   úscripts/prepare_data.pyÚslugify   s   r   ÚsizeÚintÚoverlapú	list[str]c                 C  st   |   ¡ }|sg S g d}}td|| ƒ}|t|ƒk r8d |||| … ¡ ¡ }|r.| |¡ ||7 }|t|ƒk s|S )z<Approximate token chunking by words (fast, dependency-free).r   r   ú N)ÚsplitÚmaxÚlenÚjoinr   Úappend)r   r   r   ZwordsÚchunksÚstartÚstepÚchunkr   r   r   Úchunk_words#   s   

ür#   Úpathr   ú
list[dict]c              	   C  sØ   | j }dg fg}| jdd� ¡ D ]!}t | ¡ ¡}|r(| t| d¡ƒg f¡ q|d d  |¡ qg }|D ]3\}}	d 	|	¡ ¡ }
|
sDq6t
t|
||ƒƒD ]\}}|rWd|› �nd	}| ||› d
|› |› �dœ¡ qLq6|S )z;Split markdown into (heading-anchored) chunks for citation.Zoverviewúutf-8©Úencodingé   éÿÿÿÿr   Ú
r   Ú ú#©r   ÚsourceN)ÚstemÚ	read_textÚ
splitlinesÚ_HEADING_REÚmatchr   r   r   Úgroupr   Ú	enumerater#   )r$   r   r   r0   ZsectionsÚlineÚmÚoutZanchorÚbodyr   Úir"   Úsuffixr   r   r   Úparse_markdown2   s"   
 þr=   c                   s,   | j ‰ ‡ fdd„tt| jdd�||ƒƒD ƒS )Nc                   s$   g | ]\}}|ˆ › d |› �dœ‘qS )z#part-r.   r   )Ú.0r;   r"   ©r0   r   r   Ú
<listcomp>I   s    ÿÿzparse_text.<locals>.<listcomp>r&   r'   )r0   r6   r#   r1   )r$   r   r   r   r?   r   Ú
parse_textG   s   
þrA   c                 C  sŒ   g }t | ddd��3}tt |¡ƒD ]"\}}| d¡pd ¡ }|r3| || d¡p/| j› d|› �dœ¡ qW d   ƒ |S 1 s?w   Y  |S )Nr,   r&   )Únewliner(   r   r/   z#row-r.   )Úopenr6   ÚcsvZ
DictReaderÚgetr   r   r0   )r$   r9   Úfr;   Úrowr   r   r   r   Ú	parse_csvO   s   &€ý
ÿûrH   c              	   C  sv   g }| j dd� ¡ D ].}| ¡ }|sq
t |¡}| d¡| d¡}}|r8|r8| | ¡ | ¡ | dd¡dœ¡ q
|S )Nr&   r'   ÚquestionÚanswerr/   r,   )rI   rJ   r/   )r1   r2   r   ÚjsonÚloadsrE   r   )r$   Zpairsr7   ÚobjÚqÚar   r   r   Úload_qaY   s   
"€rP   ÚpairÚdictc                 C  sZ   | d }|   d¡}|rd|› d�|vr|› d|› d�}ddtdœd	| d
 dœd|dœgiS )NrJ   r/   ú[ú]z [ÚmessagesÚsystem)ZroleZcontentÚuserrI   Z	assistant)rE   r   )rQ   rJ   r   r   r   r   Úto_sft_examplef   s   
ýÿrX   ÚrowsÚNonec                 C  sf   | j jddd� t| ddd��}|D ]}| tj|dd�d ¡ qW d   ƒ d S 1 s,w   Y  d S )	NT)ÚparentsÚexist_okÚwr&   r'   F)Zensure_asciir+   )ÚparentÚmkdirrC   ÚwriterK   Údumps)r$   rY   rF   Úrr   r   r   Úwrite_jsonlu   s   ÿ"ÿrc   c               	   C  s2  t  ¡ } | jddd� |  ¡ }t|jƒ}| d¡}t| dd¡ƒ}t| dd¡ƒ}t	| d	d
¡ƒ}g }g }t
| d¡ƒD ]P}	|	 ¡ sI|	j ¡ dkrJq<|	j ¡ }
|
dkr[| t|	ƒ¡ q<|
dkri| t|	||ƒ¡ q<|
dkrw| t|	||ƒ¡ q<|
dkrƒ| t|	ƒ¡ q<td|	j› d�ƒ q<|rštdtt|ƒ| ƒƒnd}|r§tdt|ƒ| ƒnd}g g }}t|ƒD ]\}}|rÀ|| dkrÀ|n| |¡ q²t| d¡dd„ |D ƒƒ t| d¡|ƒ | d¡d }t||ƒ tdt|ƒ› dt|ƒ› d| d¡› �ƒ tdt|ƒ› d |› �ƒ |�std!ƒ |�std"ƒ d S d S )#Nz--configzconfig/config.yaml)Údefaultzdata.raw_dirzdata.chunk_tokensi�  zdata.chunk_overlapé<   zdata.eval_splitg¸…ëQ¸¾?Ú*z	readme.mdz.jsonlz.mdz.txtz.csvz  (skipping unsupported file: ú)r   r   r)   zdata.sft_trainc                 S  s   g | ]}t |ƒ‘qS r   )rX   )r>   Úpr   r   r   r@       s    zmain.<locals>.<listcomp>zdata.sft_evalzdata.kb_dirzchunks.jsonlzSFT:  z train, z
 eval  -> zRAG:  z chunks               -> uE     ! No Q&A pairs found â€” add a .jsonl to data/raw/ for fine-tuning.uE     ! No KB documents found â€” add .md/.txt/.csv to data/raw/ for RAG.)ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsr   ZconfigÚresolver   rE   ÚfloatÚsortedZrglobÚis_dirÚnamer   r<   ÚextendrP   r=   rA   rH   Úprintr   r   r6   r   rc   )ZapÚargsZcfgZraw_dirr   r   Z
eval_splitZqa_pairsZ	kb_chunksr$   ÚextZn_evalZstrideZtrainZevalsetr;   rQ   Zkb_pathr   r   r   Úmain|   sN   



 
(ÿrv   Ú__main__)r   r	   r
   r	   )r   r	   r   r   r   r   r
   r   )r$   r   r   r   r   r   r
   r%   )r$   r   r
   r%   )rQ   rR   r
   rR   )r$   r   rY   r%   r
   rZ   )r
   rZ   ) Ú__doc__Z
__future__r   ri   rD   rK   r   ÚsysZpathlibr   r$   Úinsertr	   Ú__file__rm   r[   Zagent.promptr   Zagent.settingsr   Úcompiler3   r   r#   r=   rA   rH   rP   rX   rc   rv   Ú__name__r   r   r   r   Ú<module>   s0    $










2
ÿ