o
    guE                     @   s(  d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlZd dlmZmZ d d	lmZmZmZmZmZmZmZmZmZ d d
lmZ d dlZd dl mZ! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z. edZ/dee/ee/ f dee/ fddZ0dedefddZ1de%deddfddZ2de%dedeed ed ed  f ddfd!d"Z3de%deddfd#d$Z4de%deddfd%d&Z5de%deddfd'd(Z6de%deddfd)d*Z7de%ddfd+d,Z8e.j9fd-e#j:d.e#j:d/ee!j;e!j;ge<f deeee=  eee<  f fd0d1Z>d2e=ddfd3d4Z?	5	6			dMde%ded7e=d8e<d9ee#j@ d:eee=  d;ee#j: ddfd<d=ZAd>e#jBd7e=ddfd?d@ZCdAede=fdBdCZDdDedEee% dFeEddfdGdHZFdDedEee% deee=f fdIdJZGdDedEee% dFeEddfdKdLZHdS )N    N)UUID)create_topic_name)System)get_sql)SqliteDB)sleep)NormalizedRecordSet	RecordSet)	CallableOptionalTupleUnionListTypeVarcastAnyDict)Literal)types)
Collection)note)InvalidArgument)Table	functions)distance_functionsTvaluereturnc                 C   s$   | du rt dt| tr| S | gS )z*Wrap a value in a list if it is not a listNzvalue cannot be None)r   
isinstancer   )r    r   X/var/www/visachat/venv/lib/python3.10/site-packages/chromadb/test/property/invariants.pywrap   s
   
r!   
record_setc                 C   s(  | d du r	d}net | d trj| d dusJ t| d dkratdd | d D r4ttj| d }n:tdd | d D rHttj| d }n&tdd | d D r]ttj| d g}ntdttj| d }ntd	t| d
 | d durt| d nd| d durt| d |dS d|dS )z9Ensure that an embedding set has lists for all its values
embeddingsNr   c                 s   s    | ]}t |tV  qd S N)r   list.0	embeddingr   r   r    	<genexpr>/   s    

zwrap_all.<locals>.<genexpr>c                 s   s    | ]	}t |tjV  qd S r$   )r   npndarrayr&   r   r   r    r)   3   s
    

c                 s   s&    | ]}t |tttjtjfV  qd S r$   )r   intfloatr*   integerfloatingr'   er   r   r    r)   9   s
    
z-an embedding must be a list of floats or intszVembeddings must be a list of lists, a list of numpy arrays, a list of numbers, or Noneids	documents	metadatas)r2   r3   r4   r#   )	r   r%   lenallr   r   
Embeddingsr   r!   )r"   embedding_listr   r   r    wrap_all&   sF   

r9   
collectionc                 C   s(   |   }t|}|t|d ksJ dS )z?The given collection count is equal to the number of embeddingsr2   N)countr9   r5   )r:   r"   r;   normalized_record_setr   r   r    r;   U   s   r;   r<   
field_namer3   r4   r#   c                    s  | j |d |gddd t|d D  | }t|d dkr<|dkr4ttjt |jdks2J dS |g ks:J dS |dusBJ tt| fdd	d
}dd |D }|| }|du rgdgt|d  }|dkr{t	
t	|t	|syJ dS ||ksJ dS )zq
    The actual embedding field is equal to the expected field
    field_name: one of [documents, metadatas]
    r2   r2   includec                 S      i | ]\}}||qS r   r   r'   iidr   r   r    
<dictcomp>j       z"_field_matches.<locals>.<dictcomp>r   r#   Nc                    s    d | d   S )Nr2   r   r   )index_and_field_valueembedding_id_to_indexresultr   r    <lambda>y   s    z _field_matches.<locals>.<lambda>keyc                 S   s   g | ]\}}|qS r   r   )r'   _field_valuer   r   r    
<listcomp>}       z"_field_matches.<locals>.<listcomp>)get	enumerater5   r   nptNDArrayr   sizesortedr*   allclosearray)r:   r<   r=   actual_fieldsorted_fieldfield_valuesexpected_fieldr   rG   r    _field_matches\   s*    r]   c                    s\   t |}| j|d g dd }dd t|d D  t| fddd}||d ks,J dS )	z5The actual embedding ids is equal to the expected idsr2   r>   c                 S   r@   r   r   rA   r   r   r    rD      rE   zids_match.<locals>.<dictcomp>c                    s    |  S r$   r   rC   rH   r   r    rJ      s    zids_match.<locals>.<lambda>rK   N)r9   rQ   rR   rV   )r:   r"   r<   
actual_idsr   r_   r    	ids_match   s
   ra   c                 C      t |}t| |d dS )z?The actual embedding metadata is equal to the expected metadatar4   Nr9   r]   r:   r"   r<   r   r   r    metadatas_match      re   c                 C   rb   )AThe actual embedding documents is equal to the expected documentsr3   Nrc   rd   r   r   r    documents_match   rf   rh   c                 C   rb   )rg   r#   Nrc   rd   r   r   r    embeddings_match   rf   ri   c                 C   s(   |   d }t|tt|ksJ d S )Nr2   )rQ   r5   set)r:   r2   r   r   r    no_duplicates   s   rk   querytargetsdistance_fnc                    sN   t j| t jd}t j|t jdt  fddd|}t | | fS )zGReturn the ordered indices and distances from each query to each target)dtypec                    s   t  d| S )N   )r*   apply_along_axis)rl   rn   
np_targetsr   r    rJ      s    z"_exact_distances.<locals>.<lambda>rp   )r*   rX   float32rq   argsorttolist)rl   rm   rn   np_query	distancesr   rr   r    _exact_distances   s   ry   threadpool_sizec                 C   s   t  }| }d}d}tdd |D d | kr>||k r>t  | }|d7 }td tdd |D d | kr>||k stdd |D d | ksMJ dS )zs
    Checks that the open file descriptors are not exceeding the threadpool size
    works only for SegmentAPI
       r   c                 S      g | ]
}d |j v r|j qS sqlite3pathr'   pr   r   r    rO          z4fd_not_exceeding_threadpool_size.<locals>.<listcomp>rp   c                 S   r|   r}   r   r   r   r   r    rO      r   N)psutilProcess
open_filesr5   gccollectr   )rz   current_processr   max_retriesretry_countr   r   r     fd_not_exceeding_threadpool_size   s   "r   rp   Gz?	n_results
min_recallembedding_functionquery_indicesquery_embeddingsc              	      s  t |}t|d dkrdS |d   duot dk}|s;|dus$J |d dus,J t|d ts5J ||d  tj}	d}
| jdusGJ  dusMJ d| jv r| jd }tdd	  D sbJ t d }|
t	d
t
t| }
|dkr|tj}	|dkrtj}	|du r|du r n fdd|D }|d |durdurfdd|D t| |	d\}}| j|r|nd|snd|g dd}t|| |d dusJ |d dusJ |d dusJ |d dusJ |d dusJ dd t|d D }d}tt||D ]\}\}}t|d |d|  }|tt|t|d |  7 }t|d | D ]p\}}||v}|| }tj|| |d | | |
d}|r\|rZ|d8 }nq3|saJ t | |d | | srJ |d dur|d | |d | | ksJ |d dur|d | |d | | ksJ q3qt|d }|| | }ztd| d| d| d|
  W n
 ty   Y nw ||ksJ |d D ]}tt||sJ qdS )zBValidate that the API performs nearest_neighbor searches correctlyr2   r   Nr#   r3   gư>z
hnsw:spacec                 s   s     | ]}t |ttjfV  qd S r$   )r   r%   r*   r+   r0   r   r   r    r)      s    zann_accuracy.<locals>.<genexpr>
   cosineipc                       g | ]} | qS r   r   r'   rB   )r#   r   r    rO   	  rP   z ann_accuracy.<locals>.<listcomp>c                    r   r   r   r   )query_documentsr   r    rO     rP   )rn   )r#   r3   r4   rx   )r   query_textsr   r?   rx   r4   c                 S   r@   r   r   rA   r   r   r    rD   $  rE   z ann_accuracy.<locals>.<dictcomp>)atolrp   z
# recall: z
, missing z out of z, accuracy threshold )r9   r5   r   r%   r   l2metadatar6   mathpowr,   log10r   r   ry   rl    _query_results_are_correct_shaperR   zipr*   rX   rj   rW   r   r   sort)r:   r"   r   r   r   r   r   r<   have_embeddingsdistance_functionaccuracy_thresholdspacedimindicesrx   query_resultsid_to_indexmissingrB   	indices_idistances_iexpected_idsjrC   unexpected_idindexcorrect_distancerU   recalldistance_resultr   )r#   r   r    ann_accuracy   s   






 

"



 r   r   c                    s<   dD ]}| | d usJ t  fdd| | D sJ qd S )N)rx   r#   r3   r4   c                 3   s    | ]	}t | kV  qd S r$   )r5   )r'   rI   r   r   r    r)   b  s    
z3_query_results_are_correct_shape.<locals>.<genexpr>)r6   )r   r   result_typer   r   r    r   ]  s   r   sqlitec                 C   s   t d}|  |}|  '}t|t|j| 	 \}}|
||}tt| d W  d    S 1 s9w   Y  d S )Nembeddings_queuer   )r   querybuilderfrom_txr   selectr   Countseq_idparameter_formatexecuter   r,   fetchone)r   tqcursqlparamsrI   r   r   r    _total_embedding_queue_log_sizeg  s   
$r   systemcollectionshas_collection_mutatedc                 C   sr   |  t}|r/t|dksJ tdd |D }tdd |D }t|d || ks-J d S t|dks7J d S )Nrp   c                 s   ,    | ]}|j d ur|j ddndV  qd S )Nzhnsw:sync_thresholdi  r   rQ   r'   r:   r   r   r    r)   }      

z%log_size_below_max.<locals>.<genexpr>c                 s   r   Nzhnsw:batch_sized   r   r   r   r   r    r)     r   r   )instancer   r   sum)r   r   r   r   sync_threshold_sumbatch_size_sumr   r   r    log_size_below_maxs  s   


r   c                    s   |  t}td}| |}| jd | jd fdd|D }| 9}t|	|j
t|jd| \}}|||}	i }
|	 D ]}|d |
||d  j< qL|
W  d    S 1 sfw   Y  d S )	Nr   	tenant_idtopic_namespacec                    s   i | ]
}t  |j|qS r   )r   rC   r   _tenant_topic_namespacer   r    rD     s    zB_total_embedding_queue_log_size_per_collection.<locals>.<dictcomp>topicrp   r   )r   r   r   r   r   settingsrequirer   r   r   r   r   r   r   groupbyr   r   fetchallrC   )r   r   r   r   r   topic_mappingsr   r   r   rI   outresr   r   r    ._total_embedding_queue_log_size_per_collection  s&   

$r   c                    s   |  t}|rBt|dksJ dd |D fdd|D t| | t  t ks2J t fdd|D s@J d S t|dksJJ d S )Nrp   c                 S   s,   i | ]}|j |jd ur|jddndqS r   )rC   r   rQ   r   r   r   r    rD     s    
z;log_size_for_collections_match_expected.<locals>.<dictcomp>c                    s&   i | ]}|j |  |j   d  qS )rp   )rC   r;   r   )r   r   r    rD     s    c                 3   s$    | ]} |j  |j  kV  qd S r$   r^   r   )actual_sizesexpected_sizesr   r    r)     s
    
z:log_size_for_collections_match_expected.<locals>.<genexpr>r   )r   r   r   r   rj   keysr6   )r   r   r   r   r   )r   r   r   r    'log_size_for_collections_match_expected  s"   

r   )rp   r   NNN)Ir   r   uuidr   chromadb.ingest.impl.utilsr   chromadb.configr   chromadb.db.baser   chromadb.db.impl.sqliter   timer   r   !chromadb.test.property.strategiesr   r	   typingr
   r   r   r   r   r   r   r   r   typing_extensionsr   numpyr*   numpy.typingrS   chromadb.apir   chromadb.api.models.Collectionr   
hypothesisr   hypothesis.errorsr   pypikar   r   chromadb.utilsr   r   r!   r9   r;   r]   ra   re   rh   ri   rk   r   r7   	ArrayLiker-   r,   ry   r   EmbeddingFunctionr   QueryResultr   r   boolr   r   r   r   r   r   r    <module>   s    ,"
/
.



 


!

