o
    ge                     @   s  d dl mZmZmZmZmZmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZmZmZ d d
lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9Z9d dl:Z:e:;e<Z=G dd deZ>de?dee'ee(e'f eee	e' f f de/de,de,de4fddZ@de?dee'e	e' f dee(ef de/de,de,de4fddZAdS )     )	OptionalSequenceAnyTuplecast	GeneratorUnionDictList)MetadataReader)Consumer)System)RequestVersionContextSegmentInclusionExclusionOperator)SqliteDB)override)CursorParameterValueget_sql)OpenTelemetryClientOpenTelemetryGranularitytrace_method)	WhereWhereDocumentMetadataEmbeddingRecord	LogRecordSeqId	OperationUpdateMetadataLiteralValueWhereOperator)UUID)TableTables)QueryBuilderN)	Criterion)groupby)reducec                   @   s  e Zd ZU eed< eed< eed< eed< ee ed< dZ	ee ed< de
d	efd
dZedejedAddZedejedAddZedejedefddZedejededefddZedeje						dBdedee dee deee  dee dee dedee fddZded e dede!eddf fd!d"Z"ed#ejd$ee#e$d%f  dedefd&d#Z%ed'ejded(e&d)eddfd*d'Z'ed+ejded,ed-e(ddfd.d+Z)ed/ejded,ed-e(ddfd0d/Z*ed1ejded(e&ddfd2d1Z+ed3ejded(e&ddfd4d3Z,ed5ejd6ee& ddfd7d5Z-ed8ejd e ded9e.d:e.de/f
d;d8Z0ed<ejd e ded9e.d=e.d:e.de/fd>d<Z1ed?ejedAd@d?Z2dS )CSqliteMetadataSegment	_consumer_db_id_opentelemetry_client_collection_idN_subscriptionsystemsegmentc                 C   s<   | t| _| t| _|d | _|t| _|d | _	d S )Nid
collection)
instancer   r+   r   r*   r,   requirer   r-   r.   )selfr0   r1    r7   \/var/www/visachat/venv/lib/python3.10/site-packages/chromadb/segment/impl/metadata/sqlite.py__init__3   s
   
zSqliteMetadataSegment.__init__zSqliteMetadataSegment.startreturnc                 C   s.   | j r|  }| jj| j | j|d| _d S d S )N)collection_id
consume_fnstart)r.   	max_seqidr*   	subscribe_write_metadatar/   )r6   seq_idr7   r7   r8   r=   :   s   zSqliteMetadataSegment.stopc                 C   s   | j r| j| j  d S d S N)r/   r*   unsubscribe)r6   r7   r7   r8   stopE   s   zSqliteMetadataSegment.max_seqidc                 C   s   t d}| j ||j|jt| j	| j
k}t|\}}| j (}||| }|d u rA| j W  d    S | j|d W  d    S 1 sSw   Y  d S )N
max_seq_idr   )r#   r+   querybuilderfrom_selectrA   where
segment_idr   
uuid_to_dbr,   r   txexecutefetchoner*   	min_seqiddecode_seq_id)r6   tqsqlparamscurresultr7   r7   r8   r>   K   s   $zSqliteMetadataSegment.countrequest_version_contextc                 C   s   t d}| j ||jt| j| jk	t
|j}t|\}}| j }||| d }tt|W  d    S 1 sEw   Y  d S )N
embeddingsr   )r#   r+   rF   rG   rI   rJ   r   rK   r,   rH   fnCountr2   r   rL   rM   rN   r   int)r6   rW   embeddings_trR   rS   rT   rU   rV   r7   r7   r8   count^   s   $z"SqliteMetadataSegment.get_metadataTrI   where_documentidslimitoffsetinclude_metadatac              	   C   s,  t ddd\}}	}
|pd}|pd}|dk rtd|j|j|jg}|r2||	j|	j|	j|	j	|	j
g | j ||	|j|	jkj| |j}|dusS|dur| j ||j|	|j|	jk|j|jt| j| jk }|r|| |||	|}|r|| |||	|
|}|dur||jt|}||}||}||j|}n;| j ||j|jt| j| jk|j||}|dur||jt|}||j|}| j }t| |||W  d   S 1 sw   Y  dS )zQuery for embedding metadata.rX   embedding_metadataembedding_fulltext_searchl    r   zLimit cannot be negativeN) r$   
ValueErrorr2   embedding_idrA   extendkeystring_value	int_valuefloat_value
bool_valuer+   rF   rG   	left_joinonrH   orderbyrI   rJ   r   rK   r,   distinct_where_map_criterion_where_doc_criterionisinr`   ra   rL   list_records)r6   rW   rI   r^   r_   r`   ra   rb   r\   
metadata_t
fulltext_tselect_clauserR   
metadata_qembeddings_qrU   r7   r7   r8   get_metadatao   s   



&rU   rR   c           
      c   sX    t |\}}||| t|jd}t|dd }|D ]\}}	| t|	|V  qdS )zqGiven a cursor and a QueryBuilder, yield a generator of records. Assumes
        cursor returns rows in ID order.Nc                 S   s   t | d S )Nr   )r[   )rr7   r7   r8   <lambda>   s    z0SqliteMetadataSegment._records.<locals>.<lambda>)r   rM   iterrN   r'   _recordrt   )
r6   rU   rR   rb   rS   rT   cur_iteratorgroup_iterator_groupr7   r7   r8   ru      s   zSqliteMetadataSegment._recordszSqliteMetadataSegment._recordrows.c                 C   s   |d dd \}}}|st |ddS i }|D ]9}|dd \}}	}
}}|	dur-|	||< q|
dur6|
||< q|dur?|||< q|durP|dkrLd||< qd||< qt ||pVddS )zUGiven a list of DB rows with the same ID, construct a
        MetadataEmbeddingRecordr   N   )r2   metadata   TF)r   )r6   r   rb   r   rf   rA   r   rowrh   ri   rj   rk   rl   r7   r7   r8   r      s*   



z$SqliteMetadataSegment._insert_recordrecordupsertc           	   	   C   s(  t d}| j ||j|j|j|jt	| j
| jk|jt	|d d kt	| j
| jt	|d d t	| j|d }t|\}}|d }z||| d }W n! tjy~   |ro| || Y S td|d d   Y dS w |d d	 r| |||d d	  dS dS )
z2Add or update a single EmbeddingRecord into the DBrX   r   r2   
log_offsetzRETURNING idr   z!Insert of existing embedding ID: Nr   )r#   r+   rF   intocolumnsrJ   rf   rA   rI   r   rK   r,   insertencode_seq_idr   rM   rN   sqlite3IntegrityError_update_recordloggerwarning_update_metadata)	r6   rU   r   r   rQ   rR   rS   rT   r2   r7   r7   r8   _insert_record  s6   z&SqliteMetadataSegment._update_metadatar2   r   c           	      C   s~   t d}dd | D }|r6| j ||jt|k|j	t|
 }t|\}}||| | ||| dS )z0Update the metadata for a single EmbeddingRecordrc   c                 S   s   g | ]
\}}|d u r|qS rB   r7   ).0kvr7   r7   r8   
<listcomp>9  s    z:SqliteMetadataSegment._update_metadata.<locals>.<listcomp>N)r#   itemsr+   rF   rG   rI   r2   r   rh   rs   deleter   rM   _insert_metadata)	r6   rU   r2   r   rQ   	to_deleterR   rS   rT   r7   r7   r8   r   3  s   z&SqliteMetadataSegment._insert_metadatac           
   
      s  t dj jjjjj	j
} D ][\}}t|tr8|tt|t|ddd}qt|trN|tt|dddt|}qt|trd|tt|dt|dd}qt|try|tt|ddt|d}qt|\}}|dd}|r || dv rt dd
 fdd	}	z|	  W dS  tjy   j jtk }t|\}} || |	  Y dS w dS )z@Insert or update each metadata row for a single embedding recordrc   NINSERTINSERT OR REPLACEchroma:documentrd   r:   c                     sL   j  jjttd } t| \}} 	|| d S )Nr   )
r+   rF   r   r   rowidri   r   r   r   rM   )rR   rS   rT   rU   r2   r   r6   rQ   r7   r8   insert_into_fulltext_search  s   
	zKSqliteMetadataSegment._insert_metadata.<locals>.insert_into_fulltext_searchr:   N)r#   r+   rF   r   r   r2   rh   ri   rj   rk   rl   r   
isinstancestrr   r   boolr[   floatr   replacerM   r   r   rG   rI   r   r   )
r6   rU   r2   r   rR   rh   valuerS   rT   r   r7   r   r8   r   G  s   

	

	z$SqliteMetadataSegment._delete_recordc                 C   sd  t d}t d}| j ||jt| j| jk|j	t|d d k
 }| j |
 |j| j ||j|jt| j| jk|j	t|d d k}|jt|  t|\}}|d }||| }	|	du rtd|d d   dS |	d }
t d	}| j ||jt|
k
 }t|\}}||| dS )
z+Delete a single EmbeddingRecord from the DBrX   rd   r   r2    RETURNING idNz$Delete of nonexisting embedding ID: r   rc   )r#   r+   rF   rG   rI   rJ   r   rK   r,   rf   r   r   rs   rH   r2   rM   r   rN   r   r   )r6   rU   r   rQ   fts_trR   q_ftsrS   rT   rV   r2   rv   r7   r7   r8   _delete_record  sP   z$SqliteMetadataSegment._update_recordc           	      C   s   t d}| j ||jt| j|d |j	t| j
| jk|jt|d d k}t|\}}|d }||| }|du rVtd|d d   dS |d }|d d	 rm| |||d d	  dS dS )
z)Update a single EmbeddingRecord in the DBrX   r   r   r2   r   Nz$Update of nonexisting embedding ID: r   r   )r#   r+   rF   updatesetrA   r   r   rI   rJ   rK   r,   rf   r   rM   rN   r   r   r   )	r6   rU   r   rQ   rR   rS   rT   rV   r2   r7   r7   r8   r     s$   z%SqliteMetadataSegment._write_metadatarecordsc              	   C   s"  | j  }|D ]C}|d d tjkr| ||d q|d d tjkr,| ||d q|d d tjkr<| || q|d d tjkrK| 	|| q| j 
 tdddt| j | jt| j |d }t|\}}|d	d
}||| W d   dS 1 sw   Y  dS )zWrite embedding metadata to the database. Care should be taken to ensure
        records are append-only (that is, that seq-ids should increase monotonically)r   	operationFTrE   rJ   rA   r   r   r   N)r+   rL   r   ADDr   UPSERTDELETEr   UPDATEr   rF   r   r#   r   r   r   rK   r,   r   r   r   rM   )r6   r   rU   r   rR   rS   rT   r7   r7   r8   r@     s.   
	"z*SqliteMetadataSegment._where_map_criterionrv   r\   c           
   	      s   g }|  D ]\\}}|dkr* fddttt |D }|tdd | q|dkrJ fddttt |D }|tdd | qttttt	tf f |}	|t
||	  qtd	d |S )
N$andc                       g | ]
} | qS r7   rq   r   wr\   rv   rR   r6   r7   r8   r         z>SqliteMetadataSegment._where_map_criterion.<locals>.<listcomp>c                 S      | |@ S rB   r7   xyr7   r7   r8   r}         z<SqliteMetadataSegment._where_map_criterion.<locals>.<lambda>$orc                    r   r7   r   r   r   r7   r8   r     r   c                 S      | |B S rB   r7   r   r7   r7   r8   r}     r   c                 S   r   rB   r7   r   r7   r7   r8   r}      r   )r   r   r   r   appendr(   r   r    r	   r!   _where_clause)
r6   rR   rI   rv   r\   clauser   r   criteriaexprr7   r   r8   rq   	  s   z*SqliteMetadataSegment._where_doc_criterionrw   c                    s  |  D ]\}}|dkr' fddttt |D }tdd |  S |dkrF fddttt |D }tdd |  S |d	v r~tt|}d
| d
}	j 	j
jt|	}
|dkrv j|
  S  j|
  S td| td)Nr   c              	         g | ]} | qS r7   rr   r   r\   rw   rv   rR   r6   r7   r8   r   /      
z>SqliteMetadataSegment._where_doc_criterion.<locals>.<listcomp>c                 S   r   rB   r7   r   r7   r7   r8   r}   5  r   z<SqliteMetadataSegment._where_doc_criterion.<locals>.<lambda>r   c              	      r   r7   r   r   r   r7   r8   r   7  r   c                 S   r   rB   r7   r   r7   r7   r8   r}   =  r   )	$containsz$not_contains%r   zUnknown where_doc operator zEmpty where_doc)r   r   r   r   r(   r   r+   rF   rG   rH   r   rI   ri   liker   r2   rs   notinre   )r6   rR   rI   rv   rw   r\   r   r   r   search_termsqr7   r   r8   rr   "  s4   


zSqliteMetadataSegment.deletec                 C   sb  t d}t d}t d}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j }|jt|  |jt|  |jt|  W d    d S 1 sw   Y  d S )NrX   rc   rd   )r#   r+   rF   rG   r   rI   r2   rs   rH   rJ   r   rK   r,   r   rL   rM   r   )r6   rQ   t1t2q0rR   r   rU   r7   r7   r8   r   Q  sd   "r   )NNNNNT)3__name__
__module____qualname__r   __annotations__r   r"   r   r   r/   r   r   r9   r   r   ALLr   r=   rD   r   r>   r   r[   r]   r   r   r   r   r   r   r{   r   r%   r   ru   r   r   r   r   r   r   r   r   r   r   r@   r#   r&   rq   rr   r   r7   r7   r7   r8   r)   +   s   
 

	



	r


"
X
3

,r)   rh   r   ry   rv   r\   r:   c                 C   sR   t |ttttfrt| ttd|i|||S tt	|
 \}}t| |||||S )zLGiven a field name, an expression, and a table, construct a Pypika Criterion$eq)r   r   r[   r   r   r   r   r!   nextr~   r   _value_criterion)rh   r   ry   rv   r\   operatorr   r7   r7   r8   r     s   	r   r   opc                 C   sj  dt dtfdd}||jt| k}t|}||s&t|try||d ry|j|j}	}
|dv r:|	|k|
|kB }ne|dkrG|	|k|
|kB }nX|dkrT|	|k|
|kB }nK|d	kra|	|k |
|k B }n>|d
krn|	|k|
|kB }n1|		||
	|B }n&t|tst|trt|d tr|j
}n|j}|dv r||k}n|	|}|dv r|j||S |j	||S )z(Creates the filter for a single operatorobjr:   c                 S   s   t | t ot | ttfS rB   )r   r   r[   r   )r   r7   r7   r8   
is_numeric  s   z$_value_criterion.<locals>.is_numericr   )r   $nez$gtz$gtez$ltz$lte)r   z$nin)objectr   rI   rh   r   r   rt   rj   rk   rs   rl   ri   r2   r   )rh   r   r   ry   rv   r\   r   sub_qp_valint_col	float_colr   colr7   r7   r8   r     s:   



r   )Btypingr   r   r   r   r   r   r   r	   r
   chromadb.segmentr   chromadb.ingestr   chromadb.configr   chromadb.typesr   r   r   chromadb.db.impl.sqliter   	overridesr   chromadb.db.baser   r   r    chromadb.telemetry.opentelemetryr   r   r   r   r   r   r   r   r   r   r    r!   uuidr"   pypikar#   r$   pypika.queriesr%   pypika.functions	functionsrY   pypika.termsr&   	itertoolsr'   	functoolsr(   r   logging	getLoggerr   r   r)   r   r   r   r7   r7   r7   r8   <module>   st   , ,
    c
	


