o
    g^                     @   s  d dl Z d dlmZmZmZmZmZmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlZd dlZd dlmZ d d	lm Z  d d
l!m"Z" d dl m#Z# d dl$m%Z% d dl&Z&d dl'Z'd dl(Z(d dl)Z*dee	ddf fddZ+dee	ddf fddZ,deeg ee	ddf f  fddZ-e j.de- dde#dee	ddf fddZ/e j.dddee fddZ0deee  fddZ1e j.de1 dde#deee ddf fdd Z2defd!d"Z3d#ed$eddfd%d&Z4de	dee d ee d'eddf
d(d)Z5dGd+e6d,e6d-e6de7fd.d/Z8dGd+ed,ed-e6de7fd0d1Z9de	dee d ee d'eddf
d2d3Z:de	dee d ee d'eddf
d4d5Z;de	dee d ee d'eddf
d6d7Z<d8ed9ej=d#edee d:eddfd;d<Z>de	dee d ee d'eddf
d=d>Z?de	dee d ee d'eddf
d?d@Z@de	d ee ddfdAdBZAde	dee d ee d'eddf
dCdDZBde	dee d ee d'eddf
dEdFZCdS )H    N)	GeneratorListCallableIteratorTypecast)SystemSettings)
ProducerFn)	OperationRecordRequestVersionContextVectorQuery	OperationScalarEncodingSegmentSegmentScopeSeqIdVector)Producer)VectorReader)LocalHnswSegment)PersistentLocalHnswSegment)test_hnsw_config)FixtureRequest)countreturnc                  c   sT    t  } tdd| d}t|}|  |V  |  tj| r(t	
|  dS dS )Fixture generator for sqlite DBTFallow_resetis_persistentpersist_directoryNtempfilemkdtempr	   r   startstopospathexistsshutilrmtree	save_pathsettingssystem r/   X/var/www/visachat/venv/lib/python3.10/site-packages/chromadb/test/segment/test_vector.pysqlite&      r1   c                  c   sT    t  } tdd| d}t|}|  |V  |  tj| r(t	
|  dS dS )r   Tr   Nr!   r+   r/   r/   r0   sqlite_persistent6   r2   r3   c                   C      t tgS N)r1   r3   r/   r/   r/   r0   system_fixturesI      r6   module)scopeparamsrequestc                 c   s    t |  V  d S r5   )nextparamr;   r/   r/   r0   r.   M   s   r.   function)r9   c                      s&   dt dtfdd  fddt D S )zGenerate a sequence of embeddings with the property that for each embedding
    (other than the first and last), it's nearest neighbor is the previous in the
    sequence, and it's second nearest neighbor is the subsequentir   c                 S   s6   t | d | d g}td|  |tjd tjd}|S )Ng?
embedding_id	embeddingencodingmetadata	operation)nparrayr   r   FLOAT32r   ADD)r@   vectorrecordr/   r/   r0   create_recordX   s   z(sample_embeddings.<locals>.create_recordc                 3   s    | ]} |V  qd S r5   r/   .0r@   rN   r/   r0   	<genexpr>c       z$sample_embeddings.<locals>.<genexpr>)intr   r   r/   r/   rQ   r0   sample_embeddingsR   s   rU   c                   C   r4   r5   )r   r   r/   r/   r/   r0   vector_readersf   r7   rV   c                 c   s    | j V  d S r5   )r=   r>   r/   r/   r0   vector_readerj   s   rW   c                   C   s"   t t dtjtjddti dS )N	test_typer   )rT   )rC   typer9   
collectionrF   
file_paths)r   uuiduuid4r   VECTORUUIDr   r/   r/   r/   r0    create_random_segment_definitiono   s   
r`   segmentseq_idc                 C   sP   t   }t   | dk r!|  |krd S t d t   | dk std| )N   g      ?zTimed out waiting for seq_id )time	max_seqidsleepTimeoutError)ra   rb   r$   r/   r/   r0   syncz   s   
rh   produce_fnsc           
      C   s   |  t}tddd}|   t }|d }|||d|dd d }|| |}	|	  t|	| |	j|ddks:J |||d|dd d }t|	| |	j|dd	ksUJ d S )
Nr   collection_versionlog_positionrZ      producercollection_idn
embeddings   request_version_context   )instancer   r   reset_stater`   r$   rh   r   )
r.   rU   rW   ri   ro   rv   segment_definitionrp   max_idra   r/   r/   r0   test_insert_and_count   sB   



r|   -C6?abepsilonc                 C   s   t | | |k S r5   )absr~   r   r   r/   r/   r0   approx_equal   s   r   c                    s   t  fddt| |D S )Nc                 3   s     | ]\}}t || V  qd S r5   )r   )rP   xyr   r/   r0   rR      s    z&approx_equal_vector.<locals>.<genexpr>)allzipr   r/   r   r0   approx_equal_vector   s   r   c                 C   sx  |  t}tddd}|   t }|d }|| |}|  ||||dd\}	}
t||
d  |j|d}t|t|	ks@J t	|dd	 d
}t
||	|
D ]\}}}|d |d ks]J t|d tt|d skJ qNdd |	dd  D }|j||d}t|dksJ t	|dd	 d
}t
||	dd  |
dd  D ]\}}}|d |d ksJ t|d tt|d sJ qd S )Nr   rj   rZ   
   ro   rp   rr   rq   rt   ru   c                 S      | d S NrC   r/   vr/   r/   r0   <lambda>       z"test_get_vectors.<locals>.<lambda>keyrC   rD   c                 S   s   g | ]}|d  qS )rC   r/   rP   er/   r/   r0   
<listcomp>       z$test_get_vectors.<locals>.<listcomp>rc   idsrv   c                 S   r   r   r/   r   r/   r/   r0   r      r   )rx   r   r   ry   r`   r$   rh   get_vectorslensortedr   r   r   r   )r.   rU   rW   ri   ro   rv   rz   rp   ra   rr   seq_idsvectorsactualexpectedrb   r   r/   r/   r0   test_get_vectors   sH   



&
r   c              	   C   sl  |  t}|   tddd}t }|d }|| |}|  ||||dd\}	}
t||
d  |	D ]R}tt|d }t	|gdd d d	|d
}|
|}t|dksSJ t|d dks]J |d d d |d kskJ |d d d d uswJ t|d d d |sJ q2dd |	D }t	|dd d d|d
}|
|}t|t|	ksJ t||	D ]\}}t|dksJ |d d |d ksJ q|	dd }dd |D }t	|dd d d|d
}|
|}t|t|ksJ t||tdt|D ]?\}}}t|dksJ |d d |	| d ksJ |d d |	|d  d ks"J |d d |	|d  d ks3J qd S )Nr   rj   rZ   d   r   rt   rD   rs   Tr   kallowed_idsoptionsinclude_embeddingsrv   rC   c                 S      g | ]	}t t|d  qS rD   r   r   r   r/   r/   r0   r         z"test_ann_query.<locals>.<listcomp>Fc                 S   r   r   r   r   r/   r/   r0   r   #  r   rm      )rx   r   ry   r   r`   r$   rh   r   r   r   query_vectorsr   r   r   range)r.   rU   rW   ri   ro   rv   rz   rp   ra   rr   r   r   rL   queryresultsr   rtest_embeddingsr@   r/   r/   r0   test_ann_query   s   





 "$r   c              	      sV  |  t}|   tddd}t }|d }|| |}|  ||||dd\}	}
t||
d  |j|ddks:J t|	d d d d d t	j
d	 t|
tsOJ |
|||d
 fddtd
D dd
 d  t||
d  |j|ddksxJ |j|	d d g|dg ksJ |j|d}t|dksJ t|dd d}t||	d
d  D ]\}}|d |d ksJ t|d tt|d sJ qtt|	d d }t|gdd d d|d}||}t|dksJ tdd |d D tdd |	d
d  D ksJ |
|||d
 fddtd
D dd
 d  t||
d  |j|ddks)J d S )Nr   rj   rZ   rc   r   rt   ru   rC   rB   rs   c                 3       | ]} V  qd S r5   r/   rP   _delete_recordr/   r0   rR   ^      ztest_delete.<locals>.<genexpr>rn      r   c                 S   r   r   r/   r   r/   r/   r0   r   q  r   ztest_delete.<locals>.<lambda>r   rD   r   Fr   c                 s       | ]}|d  V  qdS rC   Nr/   rP   r   r/   r/   r0   rR     rS   c                 s   r   r   r/   r   r/   r/   r0   rR     rS   c                 3   r   r5   r/   r   r   r/   r0   rR     r   )rx   r   ry   r   r`   r$   rh   r   r   r   DELETE
isinstancer   appendr   r   r   r   r   r   r   r   r   r   setr.   rU   rW   ri   ro   rv   rz   rp   ra   rr   r   r   r   r   rL   r   knn_resultsr/   r   r0   test_delete6  s   



	

4	r   ro   rp   rG   c                    s(   fddt dD }g }|D ]}|| || qtddd}t||d  |j|ddks2J || |t|d d td	d	gt	j
d
|d t||d  |j|ddks]J |j|d}	t|	dkskJ |j|d d g|d}	t|	d d td	d	gsJ tt|d d }
t|
gdd
d
d|d}||d }|d d |d d ksJ |d d |d d ksJ |d d |d d ksJ td	d	g}
t|
gdd
d
d|d}||d }|d d |d d ksJ |d d |d d ksJ |d d |d d ksJ d
S )z3Tests the common code paths between update & upsertc                    s   g | ]}t  qS r/   )r<   rO   rU   r/   r0   r     r   z _test_update.<locals>.<listcomp>rm   r   rj   rt   ru   rC         $@NrB   r   rD   Fr   rs   r   )r   r   submit_embeddingr   rh   r   r   rH   rI   r   rJ   r   r   array_equalr   r   r   r   )ro   rp   ra   rU   rG   rr   r   r   rv   r   rL   r   r   r/   r   r0   _test_update  sn   	
""r   c           
         s   |  t}|   t }|d }tddd}|| |}|  t||||tj t	dt
ddgtjd tjd |||d fdd	tdD d
d d }	t||	 |j|ddks\J |jdg|dg kshJ d S )NrZ   r   rj   no_such_recordr   rB   rs   c                 3   r   r5   r/   r   update_recordr/   r0   rR     r   ztest_update.<locals>.<genexpr>rn   ru   rm   r   )rx   r   ry   r`   r   r$   r   r   UPDATEr   rH   rI   r   rJ   r   rh   r   r   )
r.   rU   rW   ri   ro   rz   rp   rv   ra   rb   r/   r   r0   test_update  sD   


r   c                    s   |  t}|   t }|d }tddd}|| |}|  t||||tj t	dt
ddgtjd tjd |||d fdd	tdD d
d d }	t||	 |j|ddks\J |jdg|d}
t|
dkslJ t|
d d t
ddgs|J d S )NrZ   r   rj   r   *   rB   rs   c                 3   r   r5   r/   r   upsert_recordr/   r0   rR   +  r   ztest_upsert.<locals>.<genexpr>rn   ru   r   r   rD   )rx   r   ry   r`   r   r$   r   r   UPSERTr   rH   rI   r   rJ   r   rh   r   r   r   r   )r.   rU   rW   ri   ro   rz   rp   rv   ra   rb   resultr/   r   r0   test_upsert  sD   


$r   c                 C   s   |  t}|   t }|d }tddd}|| |}|  |j|ddks)J tdd d d tj	d}z	|
|| W d S  tyK   td Y d S w )NrZ   r   rj   ru   	not_in_dbrB   z@Unexpected error. Deleting on an empty segment should not raise.)rx   r   ry   r`   r   r$   r   r   r   r   r   BaseExceptionpytestfail)r.   rW   ro   rz   rp   rv   ra   r   r/   r/   r0   test_delete_without_add8  s,   

r   c              	      s  |  t}|   tddd}t }|d }|| |}|  ||||dd\}	}
t||
d  |j|ddks:J t|	d d d d d t	j
d	 t|
tsOJ |
|||d
 fddtd
D dd
 d  t||
d  |j|ddksxJ |j|	d d g|dg ksJ |j|d}t|dksJ t|dd d}t||	d
d  D ]\}}|d |d ksJ t|d tt|d sJ qtt|	d d }t|gdd d d|d}||}t|dksJ tdd |d D tdd |	d
d  D ksJ t|tr%tj| sJ |  tj| rJ |  d S t|trHt !t" |  W d    d S 1 sAw   Y  d S d S )Nr   rj   rZ   rc   r   rt   ru   rC   rB   rs   c                 3   r   r5   r/   r   r   r/   r0   rR   ~  r   z9test_delete_with_local_segment_storage.<locals>.<genexpr>rn   r   r   c                 S   r   r   r/   r   r/   r/   r0   r     r   z8test_delete_with_local_segment_storage.<locals>.<lambda>r   rD   r   Fr   c                 s   r   r   r/   r   r/   r/   r0   rR     rS   c                 s   r   r   r/   r   r/   r/   r0   rR     rS   )#rx   r   ry   r   r`   r$   rh   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   r'   r(   _get_storage_folderdeleter   r   raisesNotImplementedErrorr   r/   r   r0   &test_delete_with_local_segment_storageV  s   



	

4
$r   c              	      s  |  t}|   t }|d }tddd}|| |}|  ||||dd\}	}
t||
d  |j|ddks:J t|	d d d d d t	j
d	 t|
tsOJ |
|||d
 fddtd
D dd
 d  t||
d  |j|ddksxJ |j|	d d g|dg ksJ |j|d}t|dksJ t|dd d}t||	d
d  D ]\}}|d |d ksJ t|d tt|d sJ qtt|	d d }t|gdd d d|d}||}t|dksJ tdd |d D tdd |	d
d  D ksJ t|tr?|jr%tj| sJ |  tj| r#J d S tj| s0J |  tj| sAJ d S d S )NrZ   r   rj   rc   r   rt   ru   rC   rB   rs   c                 3   r   r5   r/   r   r   r/   r0   rR     r   zAtest_reset_state_ignored_for_allow_reset_false.<locals>.<genexpr>rn   r   r   c                 S   r   r   r/   r   r/   r/   r0   r     r   z@test_reset_state_ignored_for_allow_reset_false.<locals>.<lambda>r   rD   r   Fr   c                 s   r   r   r/   r   r/   r/   r0   rR     rS   c                 s   r   r   r/   r   r/   r/   r0   rR     rS   )rx   r   ry   r`   r   r$   rh   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   _allow_resetr&   r'   r(   r   )r.   rU   rW   ri   ro   rz   rp   rv   ra   rr   r   r   r   r   rL   r   r   r/   r   r0   .test_reset_state_ignored_for_allow_reset_false  s   



	

4r   )r}   )Dr   typingr   r   r   r   r   r   chromadb.configr   r	   chromadb.test.conftestr
   chromadb.typesr   r   r   r   r   r   r   r   r   chromadb.ingestr   chromadb.segmentr   r\   rd   'chromadb.segment.impl.vector.local_hnswr   2chromadb.segment.impl.vector.local_persistent_hnswr   !chromadb.test.property.strategiesr   r   	itertoolsr   r"   r&   r)   numpyrH   r1   r3   r6   fixturer.   rU   rV   rW   r`   rh   r|   floatboolr   r   r   r   r   r_   r   r   r   r   r   r   r/   r/   r/   r0   <module>   s    ,$
"

'
2
Q
_
L
,
+

[