o
    gF                     @   s  d dl mZmZmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZ d dlmZ d dlmZmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z'm(Z(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z1 d dl2m3Z3 d dl4m5Z5m6Z6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZI d dlJmKZKmLZLmMZMmNZNmOZOmPZP d dl/mQZR d dlSmTZTmUZUmVZVmWZWmXZXmYZYmZZZ d dl[m\Z\ d dl]m^Z^m_Z_ d dl`maZa d dlbZbd dlcZcd dldZdeZdeYdeXf dZeecfegZhdeiddfdd Zjd!eedeefd"d#ZkG d$d% d%eZl				d.d&eRjmd'e:d(eTe; d)eTe< d*eTe= d+eTe> deVeRjnddf fd,d-ZodS )/    )retrystop_after_attemptretry_if_exception
wait_fixed)	ServerAPI)CollectionConfigurationInternal)UserIdentity)DEFAULT_DATABASEDEFAULT_TENANTSettingsSystem)SysDB)QuotaEnforcerAction)RateLimitEnforcerAsyncRateLimitEnforcer)SegmentManager)Executor)ScanFilterLimitKNN
Projection)	CountPlanGetPlanKNNPlan)add_attributes_to_current_spanOpenTelemetryClientOpenTelemetryGranularitytrace_method)ProductTelemetryClient)Producer)
Collection__version__)InvalidDimensionExceptionInvalidCollectionExceptionVersionMismatchError)CollectionMetadataIDs
Embeddings	Metadatas	DocumentsURIsWhereWhereDocumentIncludeIncludeEnum	GetResultQueryResultvalidate_metadatavalidate_update_metadatavalidate_wherevalidate_where_documentvalidate_batch)CollectionAddEventCollectionDeleteEventCollectionGetEventCollectionUpdateEventCollectionQueryEventClientCreateCollectionEventN)OptionalSequence	GeneratorListAnyCallableTypeVar)override)UUIDuuid4)wrapsT.)bound
index_namereturnc                 C   sf   d|  }t | dk st | dkrt|td| st|d| v r't|td| r1t|d S )Na  Expected collection name that (1) contains 3-63 characters, (2) starts and ends with an alphanumeric character, (3) otherwise contains only alphanumeric characters, underscores or hyphens (-), (4) contains no two consecutive periods (..) and (5) is not a valid IPv4 address, got    ?   z'^[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]$z..z0^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$)len
ValueErrorrematch)rL   msg rU   K/var/www/visachat/venv/lib/python3.10/site-packages/chromadb/api/segment.pycheck_index_nameO   s   	rW   funcc                    s&   t  dtdtdtf fdd}|S )NargskwargsrM   c                     s   | d }|j  | i |S )Nr   )_rate_limit_enforcer
rate_limit)rY   rZ   selfrX   rU   rV   wrapperd   s   zrate_limit.<locals>.wrapper)rI   rC   )rX   r_   rU   r^   rV   r\   c   s   r\   c                       s^  e Zd ZU dZeed< eed< eed< eed< e	ed< e
ed< eed< eed	< eed
< eed< def fddZedefddZedejeefdededdfddZedejeefdededejfddZedejeefdededdfddZedejeddefdee dee dedeej fddZedejededdfd dZede fd!d"Z!ed#ejededej"fd$d#Z#ed%ejee$ddd&ee%fded'ee& d(ee' d)e(ded*ede)fd+d%Z*ed,ejee$ddee%fded'ee& d(ee' ded*ede)fd-d,Z+ed.ejee$dee%fdee ded*ede)fd/d.Z,ed0ejee$ddee%fdee dee ded*edee) f
d1d2Z-ed3ejee$ee%fded*edefd4d3Z.ed5ejee$ddee%fd6e/d7ee d8ee' ded*eddfd9d5Z0ed:ejee$ee%fdeded*eddfd;d:Z1ed<ejee$dddee%fd=e2d>e/d?e3d@ee4 dAee5 dBee6 ded*ede(fdCd<Z7edDejee$ddddee%fd>e/d=e2d?ee3 d@ee4 dAee5 dBee6 ded*ede(fdEdDZ8edFejee$dddee%fd>e/d=e2d?e3d@ee4 dAee5 dBee6 ded*ede(fdGdFZ9edHeje:e;dIdJ e<dKe=dLdMdNee$ddddddddg dOee%fd>e/d=ee2 dPee> dQee dee dee dRee dSee dTee? dUe@ded*edeAfdVdHZBedWejee$dddee%fd>e/d=ee2 dPee> dTee? ded*eddfdXdWZCedYeje:e;dZdJ e<dKe=dLdMdNee$ee%fd>e/ded*edefd[dYZDed\eje:e;d]dJ e<dKe=dLdMdNee$d^ddg d_ee%fd>e/d`e3daedPee> dTee? dUe@ded*edeEfdbd\ZFedcejee$d^ee%fd>e/ddeded*edeAf
dedcZGedefdfdgZHed|dhdiZIede(fdjdkZJedefdldmZKedefdndoZLedpejMdqejNdreOejP ddfdsdpZQdqejNdtedue(ddfdvdwZRedxejMd>e/dejNfdydxZSedzejMd>e/deTfd{dzZU  ZVS )}
SegmentAPIzHAPI implementation utilizing the new segment-based internal architecture	_settings_sysdb_manager	_executor	_producer_product_telemetry_client_opentelemetry_client
_tenant_id	_topic_nsr[   systemc                    sz   t  | |j| _| t| _| t| _| t	| _
| t| _| t| _| t| _| t| _| jt| _d S N)super__init__settingsra   requirer   rb   r   rc   r   rd   r   _quota_enforcerr    rf   r   rg   r!   re   _systemr   r[   )r]   rj   	__class__rU   rV   rm   z   s   zSegmentAPI.__init__rM   c                 C   s   t t S rk   )inttimetime_nsr]   rU   rU   rV   	heartbeat   s   zSegmentAPI.heartbeatzSegmentAPI.create_databasenametenantNc                 C   s@   t |dk r
td| jjtj||d | jjt ||d d S )NrN   z0Database name must be at least 3 characters long)actionrz   ry   )idry   rz   )	rP   rQ   rp   enforcer   CREATE_DATABASErb   create_databaserH   r]   ry   rz   rU   rU   rV   r      s   
zSegmentAPI.get_databasec                 C   s   | j j||dS N)ry   rz   )rb   get_databaser   rU   rU   rV   r      s   zSegmentAPI.delete_databasec                 C   s   | j j||d d S r   )rb   delete_databaser   rU   rU   rV   r      s   zSegmentAPI.list_databaseslimitoffsetc                 C   s   | j j|||dS )N)r   r   rz   )rb   list_databases)r]   r   r   rz   rU   rU   rV   r      s   zSegmentAPI.create_tenantc                 C   s&   t |dk r
td| jj|d d S )NrN   z.Tenant name must be at least 3 characters longry   )rP   rQ   rb   create_tenantr]   ry   rU   rU   rV   r      s
   
c                 C   s   t dttgdS )N )user_idrz   	databases)r   r
   r	   rw   rU   rU   rV   get_user_identity   s
   zSegmentAPI.get_user_identityzSegmentAPI.get_tenantc                 C   s   | j j|dS )Nr   )rb   
get_tenantr   rU   rU   rV   r      s   zSegmentAPI.create_collectionFconfigurationmetadataget_or_createdatabasec                 C   s   |d urt | t| | jjtj|||d t }t||||d ur$|nt ||d d}| j	j
|j|j| g |jd |||d	\}	}
|
rV| j|	}|D ]}| j	| qLn	td| d | jtt|d tdt|i |	S )Nr{   rz   ry   r   )r|   ry   r   r   rz   r   	dimension)	r|   ry   r   segmentsr   r   r   rz   r   Collection z/ already exists, returning existing collection.)collection_uuidr   )r4   rW   rp   r}   r   CREATE_COLLECTIONrH   CollectionModelr   rb   create_collectionr|   ry   get_configurationr   rc   #prepare_segments_for_new_collectioncreate_segmentloggerdebugrf   capturer>   strr   )r]   ry   r   r   r   rz   r   r|   modelcollcreatedr   segmentrU   rU   rV   r      s^   

z#SegmentAPI.get_or_create_collectionc                 C   s   | j |||d||dS )NT)ry   r   r   r   rz   r   )r   )r]   ry   r   r   rz   r   rU   rU   rV   get_or_create_collection  s   zSegmentAPI.get_collectionc                 C   s.   | j j|||d}|r|d S td| d)Nry   rz   r   r   r    does not exist.)rb   get_collectionsr&   r]   ry   rz   r   existingrU   rU   rV   get_collection,  s   	zSegmentAPI.list_collectionc                 C   s(   | j jtj||d | jj||||dS )N)r{   rz   r   )r   r   rz   r   )rp   r}   r   LIST_COLLECTIONSrb   r   )r]   r   r   rz   r   rU   rU   rV   list_collections>  s   
zSegmentAPI.list_collectionszSegmentAPI.count_collectionsc                 C   s   t | jj||d}|S )Nrz   r   )rP   rb   r   )r]   rz   r   collection_countrU   rU   rV   count_collectionsR  s   zSegmentAPI._modifyr|   new_namenew_metadatac                 C   s   |rt | |rt| | |}| jjtj|||d |r+|r+| jj|||d d S |r7| jj||d d S |rC| jj||d d S d S )Nr   )ry   r   r   )r   )	rW   r5   _get_collectionrp   r}   r   UPDATE_COLLECTIONrb   update_collection)r]   r|   r   r   rz   r   _rU   rU   rV   _modify`  s$   
	zSegmentAPI.delete_collectionc                 C   sT   | j j|||d}|r"| j j|d j||d | j|d j d S td| d)Nr   r   r   r   r   )rb   r   delete_collectionr|   rc   delete_segmentsrQ   r   rU   rU   rV   r     s   	zSegmentAPI._addidscollection_id
embeddings	metadatas	documentsurisc	              	   C   s   |  |}	| j|tjj t|||||fd|  i tt	tjj|||||d}
| 
|	|
 | jjtj||||||d | j||
 | jtt|t||d urWt|nd|d ur`t|nd|d urit|ndd dS )Nmax_batch_sizer   r   r   r   r   r{   rz   r   r   r   r   r   r   )r   
add_amountwith_metadatawith_documents	with_urisT)r   rc   hint_use_collectiont	OperationADDr8   get_max_batch_sizelist_records_validate_embedding_record_setrp   r}   r   re   submit_embeddingsrf   r   r9   r   rP   )r]   r   r   r   r   r   r   rz   r   r   records_to_submitrU   rU   rV   _add  sJ   



	zSegmentAPI._updatec	              
   C   s   |  |}	| j|tjj t|||||fd|  i tt	tjj|||||d}
| 
|	|
 | jjtj||||||d | j||
 | jtt|t||rUt|nd|r\t|nd|rct|nd|rjt|ndd dS )Nr   r   r   r   )r   update_amountwith_embeddingsr   r   r   T)r   rc   r   r   r   UPDATEr8   r   r   r   r   rp   r}   r   re   r   rf   r   r<   r   rP   r]   r   r   r   r   r   r   rz   r   r   r   rU   rU   rV   _update  sL   



zSegmentAPI._upsertc	              	   C   s   |  |}	| j|tjj t|||||fd|  i tt	tjj|||||d}
| 
|	|
 | jjtj||||||d | j||
 dS )Nr   r   r   T)r   rc   r   r   r   UPSERTr8   r   r   r   r   rp   r}   r   re   r   r   rU   rU   rV   _upsert
  s8   



zSegmentAPI._getc                 C   
   t | tS rk   
isinstancer'   erU   rU   rV   <lambda>:     
 zSegmentAPI.<lambda>      T)r   waitstopreraise)r   r   r   wheresortpage	page_sizewhere_documentincludec                 C   s2  t t||r
t|ndd | |}|d urt| |	d ur$t|	 | jjtj	||||	|d |d ur9t
d|rE|rE|d | }|}|rKt|nd}| jtt|||rY|ndd|
v r`|ndd|
v rg|ndd|
v rn|ndd	 | jt|t|||	t|pd|ttj|
v tj|
v tj|
v d
tj|
v S )Nr   r   	ids_count)r{   rz   r   r   r   r   zSorting is not yet supported   r   r   r   )r   r   r   include_metadatainclude_documentsinclude_urisF)r   r   rP   _scanr6   r7   rp   r}   r   GETNotImplementedErrorrf   r   r;   rd   getr   r   r   r   r1   r   r   r   r   )r]   r   r   r   r   r   r   r   r   r   r   rz   r   scan
ids_amountrU   rU   rV   _get8  s`   
	

zSegmentAPI._deletec           
      C   s^  t t||r
t|ndd |d urt| |d urt| |d u s-|d urMt|dkrM|d u s;|d urMt|dkrM|d u sI|d urMt|dkrMtd| |}| jjt	j
||||d | j|tjj
 |sm|sm|s}| jt|t|||d }n|}t|dkrd S tttjj
|d}	| |j|	 | j||	 | jtt|t|d d S )Nr   r   aU  
                You must provide either ids, where, or where_document to delete. If
                you want to delete all data in a collection you can delete the
                collection itself using the delete_collection method. Or alternatively,
                you can get() all the relevant ids and then delete them.
                )r{   rz   r   r   r   r   )	operationr   )r   delete_amount)r   r   rP   r6   r7   rQ   r   rp   r}   r   DELETErc   r   r   r   rd   r   r   r   r   r   r   
collectionre   r   rf   r   r:   )
r]   r   r   r   r   rz   r   r   ids_to_deleter   rU   rU   rV   _delete  sX   
	zSegmentAPI._countc                 C   r   rk   r   r   rU   rU   rV   r     r   c                 C   s&   t dt|i | jt| |S )Nr   )r   r   rd   countr   r   )r]   r   rz   r   rU   rU   rV   _count  s   zSegmentAPI._queryc                 C   r   rk   r   r   rU   rU   rV   r     r   
   )r   r   	distancesquery_embeddings	n_resultsc	                 C   s@  t t||t|d t|}	| jtt||	||d ur|	nd|d ur&|	ndd|v r-|	ndd|v r4|	ndd|v r;|	ndd|v rB|	ndd	 |d urOt| |d urWt| | |}
|D ]}| j	|
j
t|dd	 q^| jjtj|||||d
 | jt|
t||td ||ttj|v tj|v tj|v tj|v tj|v S )N)r   r  r   r   r   r   r   r  )	r   query_amountr  with_metadata_filterwith_document_filterinclude_metadatasr   r   include_distancesFupdate)r{   rz   r   r   r  r  )r   r   rP   rf   r   r=   r6   r7   r   _validate_dimensionr   rp   r}   r   QUERYrd   knnr   r   r   r   r1   r   r   r   r  r   )r]   r   r  r  r   r   r   rz   r   r  r   	embeddingrU   rU   rV   _query  sb   
	
zSegmentAPI._peeknc                 C   s   t dt|i | j||dS )Nr   )r   )r   r   r   )r]   r   r  rz   r   rU   rU   rV   _peek?  s   
c                 C   s   t S rk   r#   rw   rU   rU   rV   get_versionL     zSegmentAPI.get_versionc                 C   s   d S rk   rU   rw   rU   rU   rV   reset_stateP  r  zSegmentAPI.reset_statec                 C   s   | j   dS )NT)rq   r  rw   rU   rU   rV   resetT  s   
zSegmentAPI.resetc                 C   s   | j S rk   )ra   rw   rU   rU   rV   get_settingsY  s   zSegmentAPI.get_settingsc                 C   s   | j jS rk   )re   r   rw   rU   rU   rV   r   ]  s   zSegmentAPI.get_max_batch_sizez)SegmentAPI._validate_embedding_record_setr   recordsc                 C   sF   t dt|d i |D ]}|d dur | j|t|d dd qdS )zQValidate the dimension of an embedding record before submitting it to the system.r   r|   r  NTr
  )r   r   r  rP   )r]   r   r  recordrU   rU   rV   r   e  s   dimr  c                 C   s^   |d du r|r|j }| jj||d ||d< dS dS |d |kr-td| d|d  dS )zValidate that a collection supports records of the given dimension. If update
        is true, update the collection if the collection doesn't already have a
        dimension.r   N)r|   r   zEmbedding dimension z* does not match collection dimensionality )r|   rb   r   r%   )r]   r   r  r  r|   rU   rU   rV   r  t  s   zSegmentAPI._validate_dimensionzSegmentAPI._get_collectionc                 C   s6   | j j|d}|rt|dkrtd| d|d S )N)r|   r   r   r   )rb   r   rP   r&   )r]   r   collectionsrU   rU   rV   r     s   
zSegmentAPI._scanc                 C   sN   | j |}dd |d D }t|d |tjj |tjj |tjjd dS )Nc                 S   s   i | ]}|d  |qS )scoperU   ).0r   rU   rU   rV   
<dictcomp>  s    
z$SegmentAPI._scan.<locals>.<dictcomp>r   r   )r   r  r   r  )	rb   get_collection_with_segmentsr   r   SegmentScopeVECTORMETADATAr   RECORD)r]   r   collection_and_segmentsscope_to_segmentrU   rU   rV   r     s   

)rM   N)W__name__
__module____qualname____doc__r   __annotations__r   r   r   r!   r    r   r   r   r   rm   rF   rt   rx   r   r   	OPERATIONr
   r   r   Databaser   r   r?   r@   r   r   r   r   Tenantr   r\   r	   r   r(   boolr   r   r   r   r   r   rG   r   r   r)   r*   r+   r,   r-   r   r   r   r   r   r   r   r.   r/   r0   r2   r   r   r   r3   r  r  r  r  r  r  r   ALLr"   rB   OperationRecordr   r  r   r   r   __classcell__rU   rU   rr   rV   r`   l   sT  
 

 




F




"
	

4	

6	

+
	

J
K

	
	

D



r`   r   r   r   r   r   r   c                 c   s    |g krd}t |D ]M\}}d}|r|| }|r,|| }	|r(i |d|	i}nd|	i}|rA|| }
|r=i |d|
i}nd|
i}tj||durL|| ndtjj|| d}|V  qdS )ziConvert parallel lists of embeddings, metadatas and documents to a sequence of
    SubmitEmbeddingRecordsNzchroma:documentz
chroma:uri)r|   r  encodingr   r   )	enumerater   r0  ScalarEncodingFLOAT32)r   r   r   r   r   r   ir|   r   documenturir  rU   rU   rV   r     s4   r   )NNNN)ptenacityr   r   r   r   chromadb.apir   chromadb.api.configurationr   chromadb.authr   chromadb.configr	   r
   r   r   chromadb.db.systemr   chromadb.quotar   r   chromadb.rate_limitr   r   chromadb.segmentr   $chromadb.execution.executor.abstractr   &chromadb.execution.expression.operatorr   r   r   r   r   "chromadb.execution.expression.planr   r   r    chromadb.telemetry.opentelemetryr   r   r   r   chromadb.telemetry.productr    chromadb.ingestr!   chromadb.typesr"   r   chromadbr$   chromadb.errorsr%   r&   r'   chromadb.api.typesr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   !chromadb.telemetry.product.eventsr9   r:   r;   r<   r=   r>   typesr   typingr?   r@   rA   rB   rC   rD   rE   	overridesrF   uuidrG   rH   	functoolsrI   ru   loggingrR   rJ   	getLoggerr&  r   r   rW   r\   r`   r   r0  r   rU   rU   rU   rV   <module>   st    L 	$	
	      @