o
    gZ                  
   @   s  d dl Z d dlZd dlmZ d dlZd dlZd dlmZmZm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d d	lmZ d dlm   m!  mZ d dl"m   m!  m#Z# d d
lm$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/Z/d dl0Z0d dl1Z1d dl2m3Z4 d dl5m6Z6 d dl7Z8eg ef Z9eddddddde1: dgZ;ej<de;ddej=deeddf fddZ>ej?ej@deAddddddZBejCdejDde
ejEejFf fdd ZGeejHeG d!d"d# d"d# fd$d%ded&e	e
ejEejFf  ddfd'd(ZIdeddfd)d*ZJded+eKd,ejFd-eddf
d.d/ZLd0d1 ZMG d2d3 d3e&ZNd4ZOG d5d6 d6e(ZPd7ejQdeddfd8d9ZRd7ejQdeddfd:d;ZSdeddfd<d=ZTd7ejQdeddfd>d?ZUdS )@    N)
Connection)	GeneratorCallableListTuplecast)UUID)given)	ClientAPI	ServerAPI)SettingsSystem)VectorReader)LocalSegmentManager)hashing_embedding_function)EmbeddingStateMachineStatestraceEmbeddingStateMachineBase)run_state_machine_as_testruleprecondition
initializeMultipleResults)Client)DefaultEmbeddingFunctionzchromadb.api.segment.SegmentAPIz chromadb.db.impl.sqlite.SqliteDBz7chromadb.segment.impl.manager.local.LocalSegmentManagerT)chroma_api_implchroma_sysdb_implchroma_producer_implchroma_consumer_implchroma_segment_manager_implallow_resetis_persistentpersist_directorymodule)scopeparamsrequestreturnc                 c   sP    | j }|j}tj|stj|dd |V  tj|r&tj|dd d S d S )NT)exist_ok)ignore_errors)paramr"   ospathexistsmakedirsshutilrmtree)r&   configuration	save_path r3   Z/var/www/visachat/venv/lib/python3.10/site-packages/chromadb/test/property/test_persist.pysettings6   s   r5   
   with_hnsw_paramswith_persistent_hnsw_paramsmax_hnsw_batch_sizemax_hnsw_sync_thresholdcoll)keydrawc                 C   s8   | t jdtdddd}| t t|}||fS )NTr6   r7   )
strategiescollectionsstjust
recordsets)r>   
collection	recordsetr3   r3   r4   !collection_and_recordset_strategyP   s   	rF      c                 C   s
   | d j S )Nr   )name)xr3   r3   r4   <lambda>e   s   
 rJ   )min_size	unique_by)#collection_and_recordset_strategiesrM   c           	      C   s0  t | }|  t|}|  |D ]8\}}|j|j|j|jd}|j	di | t
|| t
|| t
|| t
|| t
j|||jd q|  ~~t | }|  t|}|D ].\}}|j|j|jd}t
|| t
|| t
|| t
|| t
j|||jd qa|  ~~d S )NrH   metadataembedding_function)rP   rH   rP   r3   )r   startClientCreatorfrom_systemresetcreate_collectionrH   rO   rP   add
invariantscountmetadatas_matchdocuments_match	ids_matchann_accuracystopget_collection)	r5   rM   system_1client_1collection_strategyrecordset_strategyr<   system_2client_2r3   r3   r4   test_persista   sb   

rf   c                    sn  t | }|  t|}|jddddd}|t}||jt	 dt
f fdd}| }|jdd	gd
gdggd | |ksCJ | }|jdgdggd | |ksWJ | }|jg dd
gdgdggd | |kspJ | }|jdgdggd |jdd	gd
gdggd | |ksJ | }|jdgdggd |jdd	gd
gdggd | |ksJ | }d S )Ntest   )hnsw:batch_sizehnsw:sync_threshold)rH   rO   r'   c                      s2   t d z	tj  W S  ty   Y dS w )Ng?)timesleepr+   r,   getmtime_get_metadata_fileFileNotFoundErrorr3   segmentr3   r4   get_index_last_modified_at   s   
z7test_sync_threshold.<locals>.get_index_last_modified_at12      ?       @)ids
embeddings3      @)rt   ru   rz   4      @5      @)r   rR   rS   rT   rV   instancer   get_segmentidr   floatrW   upsert)r5   systemclientrD   managerrs   last_modified_atr3   rq   r4   test_sync_threshold   s6   



r   collection_name
record_setconnc              
   C   s   z;t | }|  t|}|j|t d}t|| t	|| t
|| t|| t|| |  W d S  tyN } z|| |d }~ww )NrQ   )r   rR   rS   rT   r_   r?   "not_implemented_embedding_functionrX   rY   rZ   r[   r\   r]   r^   	Exceptionsend)r5   r   r   r   r   r   r<   er3   r3   r4   load_and_check   s&   

r   c                  C   s:   zt d} | dg | W S  ty   t d Y S w )N
forkserverchromadbspawn)multiprocessingget_contextset_forkserver_preloadr   )ctxr3   r3   r4   get_multiprocessing_context   s   
r   c                   @   s   e Zd ZdZdS )#PersistEmbeddingsStateMachineStatespersistN)__name__
__module____qualname__r   r3   r3   r3   r4   r     s    r      c                       s|   e Zd Zdedef fddZeeddej	fddZe
d	d
 e dddZdeddf fddZdddZ  ZS )PersistEmbeddingsStateMachiner   r5   c                    s.   || _ || _t| _| j   t | j  d S N)r   r5    MIN_STATE_CHANGES_BEFORE_PERSIST(min_state_changes_left_before_persistingrU   super__init__)selfr   r5   	__class__r3   r4   r     s
   
z&PersistEmbeddingsStateMachine.__init__rD   rD   c                 C   sX   | j   | j j|j|j|jd| _|j| _td | t	j
 tjg g g g d| _d S )NrN   init)rx   	metadatas	documentsry   )r   rU   rV   rH   rO   rP   rD   r   on_state_changer   r   r?   StateMachineRecordSetrecord_set_state)r   rD   r3   r3   r4   r     s   
z(PersistEmbeddingsStateMachine.initializec                 C   s   t | jd dko| jdkS )Nrx   rG   r   )lenr   r   r   r3   r3   r4   rJ   ,  s    
z&PersistEmbeddingsStateMachine.<lambda>r'   Nc                 C   sp   |  tj | jj}t \}}t }|jt	| j
|| j|fd}|  |  | r2| }||  d S )N)targetargs)r   r   r   rD   rH   r   Piper   Processr   r5   r   rR   joinpollrecvclose)r   r   conn1conn2r   pr   r3   r3   r4   r   +  s   z%PersistEmbeddingsStateMachine.persist	new_statec                    s2   t  | |tjkrt| _d S |  jd8  _d S )NrG   )r   r   r   r   r   r   )r   r   r   r3   r4   r   B  s
   
z-PersistEmbeddingsStateMachine.on_state_changec                 C   s   | j   d S r   )r   rU   r   r3   r3   r4   teardownK  s   z&PersistEmbeddingsStateMachine.teardown)r'   N)r   r   r   r
   r   r   r   collection_str?   
Collectionr   r   r   strr   r   __classcell__r3   r3   r   r4   r     s    	r   caplogc                    s,   |  tj t t fdd d S )Nc                      s   t  dS )Nr5   r   )r   r3   r   r5   r3   r4   rJ   U  s    z/test_persist_embeddings_state.<locals>.<lambda>)	set_levelloggingERRORr   r   r   )r   r5   r3   r   r4   test_persist_embeddings_stateO  s
   
r   c                 C   s8  t |}t||d}|jtjdddddddd tddtji g dd	d

d |	  |
  |  |  |  |jdgddggd gd dd\}|	  |
  |  |  |  |jddgddgddggd d gd dd\}}|	  |
  |  |  |  |j|gd |	  |  d S )Nr   A00   rh   hnsw:construction_efhnsw:search_efhnsw:Mrj   ri   z$2d3eddc7-2314-45f4-a951-47a9a8e099d2   FT
rH   rO   rP   r   	dimensiondtypeknown_metadata_keysknown_document_keywordshas_documentshas_embeddingsr   0g      ?g     ?rx   ry   r   r   r   rt   ru   g     P?g     ?g     ÿg     ?rx   )r   r   r   r   r?   r   r   npfloat16r]   rY   fields_matchlog_size_below_maxno_duplicatesadd_embeddingsdelete_by_idsr   )r   r5   r   stateembedding_ids_0embedding_ids_1embedding_ids_2r3   r3   r4   test_delete_less_than_kY  sR   
".r   c                 C   s   t | }t| |d}|jtjdddddddt tddd i g dd	d

d |jg dddgddgddggg dd dd |j	dgd |jdgddggd gd dd |
  d S )Nr   r   r   rh   )r   r   r   ri   rj   z$0851f751-2f11-4424-ab23-4ae97074887ar   FTr   r   )r   rt   ru   r   )NNNr   r   r   r   rG   )r   r   r   r   r?   r   r   r   r   r   r   )r5   r   r   r3   r3   r4   test_delete_add_after_persist  sJ   

r   c                 C   sn  t |}t||d}|jtjdddddddtdtjdt	d	dtji g d
dd
d |
  |  |  |  |  ttt |jg dg dg dg dg dg dg dg dgd d dd\}}}}}}	}
|
  |  |  |  |  td |jdgg dgd d dd}|
  |  |  |  |  ttt |jg dg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d dd\}}}}}}}}}}}}}}}}}|
  |  |  |  |  |jd-d.gg d/g d0gd d gd dd |
  |  |  |  |  |jg d1g d2g d3g d4g d5g d6g d7gd d dd |
  |  |  |  |  |  |
  |  |  |  |  |jd8g d9gd d dd |
  |  |  |  |  |jd:d;gg d<g d=gd d gd dd |
  |  |  |  |  |jd>d?d@dA|dBdCdDdE||dFdGdH|dIdJgg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d dd |
  |  |  |  |  d S )KNr   zJqzMs4pPm14c
r   	      r   \   )dimr   z$45c5c816-0a90-4293-8d01-4325ff860040FTr   r   )Ne8r6r|   YaoqFjA2cjHCvru   )        r   r   )rv   rv   rv   )rw   rw   rw   )r{   r{   r{   )r}   r}   r}   )r   r   r   )      @r   r   r   r   z

	MVu393QTc)      @r   r   )CyF0Mk-q_Fwu2D2sQSFogDgPLkcfTSrwuQHQ6w4f51qWr2enLPQw8uKYs1Gwdzt5W8tpsnfJbV7zr~   V1iFkoJXZw4uFc7vEEwrPYf)       @r  r  )      "@r  r  )      $@r  r  )      &@r	  r	  )      (@r
  r
  )      *@r  r  )      ,@r  r  )      .@r  r  )      0@r  r  )      1@r  r  )      2@r  r  )      3@r  r  )      4@r  r  )      5@r  r  )      6@r  r  )      7@r  r  )      8@r  r  r   df_RWhR0HelOcv)      9@r  r  )      :@r  r  )3R9_44u3BMZCXZDSUelx)      ;@r   r   )      <@r!  r!  )      =@r"  r"  )      >@r#  r#  )      ?@r$  r$  )      @@r%  r%  YlVm)     @@r'  r'  Rk1TPL)      A@r*  r*  )     A@r+  r+  r   r   r   r   r   r   r   r   r   r  r  r  r  )r   r   r   r   r?   r   r   r   float64r   r]   rY   r   r   r   r   r   r   r   printr   r   )r   r5   r   r   r   r   r   embedding_ids_3embedding_ids_4embedding_ids_5embedding_ids_6_embedding_ids_12embedding_ids_17embedding_ids_18embedding_ids_22r3   r3   r4   Mtest_batch_size_less_than_sync_with_duplicate_adds_results_in_skipped_seq_ids  s  
		0,r7  )Vr   r   multiprocessing.connectionr   multiprocessing.contextrl   typingr   r   r   r   r   uuidr   
hypothesisr	   hypothesis.strategiesr?   rA   pytestr   chromadb.apir
   r   chromadb.configr   r   chromadb.segmentr   #chromadb.segment.impl.manager.localr   !chromadb.test.property.strategiesrg   property!chromadb.test.property.invariantsrX   r   &chromadb.test.property.test_embeddingsr   r   r   hypothesis.statefulr   r   r   r   r   r+   r/   tempfilechromadb.api.clientr   rS   "chromadb.utils.embedding_functionsr   numpyr   CreatePersistAPImkdtempconfigurationsfixtureFixtureRequestr5   sharedr@   rB   r   	compositeDrawFnr   	RecordSetrF   listsrf   r   r   r   r   r   r   r   LogCaptureFixturer   r   r   r7  r3   r3   r3   r4   <module>   s     @;
<


23