o gX@sUdZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZmZddlZddlmZmZddlmZddlmZdd lmZd d lmZd d lmZmZmZm Z m!Z!m"Z"m#Z#d dlm$Z$d dl%m&Z&d dl'm(Z(d dl)m*Z*e$+e,Z-dZ.dZ/ej0dej1dZ2GdddeZ3GdddeZ4dej5fddZ6e gej5fZ7e6a8e7e9d<e6fde7ddfdd Z:dej5fd!d"Z;dAd#d$Zd)d d*ej?ej@fe jAd+d,e*d-eBd.e=d/eCd0eCd1eeeDeeeDd2ffd3ee=ee=d2ffdefd4d5ZEd-eBd6e eBdeBfd7d8ZFdBd9ed:e eBddfd;d<ZGd=ee!d>eBd9ede!fd?d@ZHdS)Cz>Contains utilities to handle HTTP requests in Huggingface Hub.N) lru_cache) HTTPStatus)CallableOptionalTupleTypeUnion) HTTPErrorResponse) HTTPAdapter)PreparedRequest)OfflineModeIsEnabled) constants)BadRequestErrorDisabledRepoErrorEntryNotFoundErrorGatedRepoErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError)logging)JSONDecodeError) SliceFileObj) HTTP_METHOD_TX-Amzn-Trace-Idz x-request-ida  # staging or production endpoint ^https://[^/]+ ( # on /api/repo_type/repo_id /api/(models|datasets|spaces)/(.+) | # or /repo_id/resolve/revision/... /(.+)/resolve/(.+) ) )flagscs6eZdZdZfddZdedeffdd ZZS)UniqueRequestIdAdapterrc stj|fi|t|jvr|jtptt|jt<t|jdd d}t d|jtd|j d|j d|d dS) N authorizationz Bearer hf_zRequest z:  z (authenticated: ))super add_headersX_AMZN_TRACE_IDheadersget X_REQUEST_IDstruuiduuid4 startswithloggerdebugmethodurl)selfrequestkwargs has_token __class__R/var/www/visachat/venv/lib/python3.10/site-packages/huggingface_hub/utils/_http.pyr$Ms &z"UniqueRequestIdAdapter.add_headersr2returnc shztj|g|Ri|WStjy3}z|jt}|dur.g|jd|dR|_d}~ww)zSCatch any RequestException to append request id to the error message for debugging.Nz (Request ID: r")r#sendrequestsRequestExceptionr&r'r%args)r1r2r=r3e request_idr5r7r8r:Zs zUniqueRequestIdAdapter.send) __name__ __module__ __qualname__r%r$r r r: __classcell__r7r7r5r8rJs  rc@seZdZdedefddZdS)OfflineAdapterr2r9cOstd|jd)Nz Cannot reach za: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable.)r r0)r1r2r=r3r7r7r8r:gs zOfflineAdapter.sendN)r@rArBr r r:r7r7r7r8rDfsrDr9cCsNt}tjr|dt|dt|S|dt|dt|S)Nzhttp://zhttps://)r;SessionrHF_HUB_OFFLINEmountrDr)sessionr7r7r8_default_backend_factorymsrI_GLOBAL_BACKEND_FACTORYbackend_factorycCs|atdS)a Configure the HTTP backend by providing a `backend_factory`. Any HTTP calls made by `huggingface_hub` will use a Session object instantiated by this factory. This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications). Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. Example: ```py import requests from huggingface_hub import configure_http_backend, get_session # Create a factory function that returns a Session with configured proxies def backend_factory() -> requests.Session: session = requests.Session() session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} return session # Set it as the default session factory configure_http_backend(backend_factory=backend_factory) # In practice, this is mostly done internally in `huggingface_hub` session = get_session() ``` N)rJreset_sessions)rKr7r7r8configure_http_backend|s rMcCstttdS)a Get a `requests.Session` object, using the session factory from the user. Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. Example: ```py import requests from huggingface_hub import configure_http_backend, get_session # Create a factory function that returns a Session with configured proxies def backend_factory() -> requests.Session: session = requests.Session() session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} return session # Set it as the default session factory configure_http_backend(backend_factory=backend_factory) # In practice, this is mostly done internally in `huggingface_hub` session = get_session() ```  process_id thread_id)_get_session_from_cacheosgetpid threading get_identr7r7r7r8 get_sessionsrVcCs tdS)zReset the cache of sessions. Mostly used internally when sessions are reconfigured or an SSLError is raised. See [`configure_http_backend`] for more details. N)rQ cache_clearr7r7r7r8rLs rLrOrPcCstS)z Create a new session per thread using global factory. Using LRU cache (maxsize 128) to avoid memory leaks when using thousands of threads. Cache is cleared when `configure_http_backend` is called. )rJrNr7r7r8rQsrQ) max_retriesbase_wait_time max_wait_timeretry_on_exceptionsretry_on_status_codesr/r0rZr[r\r].r^c Kspt|tr|f}t|tr|f}d}|} d} d|vr*t|dtjtfr*|d} t} |d7}z:| dur>|d| | j d||d|} | j |vrQ| WSt d| j d|d |||krk| | WSWn/|y} z#t d | d |d |t| tjrt||kr| WYd} ~ nd} ~ wwt d | d |d|dt| t|| d} q.)a# Wrapper around requests to retry calls on an endpoint, with exponential backoff. Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) and/or on specific status codes (ex: service unavailable). If the call failed more than `max_retries`, the exception is thrown or `raise_for_status` is called on the response object. Re-implement mechanisms from the `backoff` library to avoid adding an external dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. Args: method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): HTTP method to perform. url (`str`): The URL of the resource to fetch. max_retries (`int`, *optional*, defaults to `5`): Maximum number of retries, defaults to 5 (no retries). base_wait_time (`float`, *optional*, defaults to `1`): Duration (in seconds) to wait before retrying the first time. Wait time between retries then grows exponentially, capped by `max_wait_time`. max_wait_time (`float`, *optional*, defaults to `8`): Maximum duration (in seconds) to wait before retrying. retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*): Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. By default, retry on `requests.Timeout` and `requests.ConnectionError`. retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `503`): Define on which status codes the request must be retried. By default, only HTTP 503 Service Unavailable is retried. **kwargs (`dict`, *optional*): kwargs to pass to `requests.request`. Example: ``` >>> from huggingface_hub.utils import http_backoff # Same usage as "requests.request". >>> response = http_backoff("GET", "https://www.google.com") >>> response.raise_for_status() # If you expect a Gateway Timeout from time to time >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) >>> response.raise_for_status() ``` When using `requests` it is possible to stream data by passing an iterator to the `data` argument. On http backoff this is a problem as the iterator is not reset after a failed call. This issue is mitigated for file objects or any IO streams by saving the initial position of the cursor (with `data.tell()`) and resetting the cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff will fail. If this is a hard constraint for you, please let us know by opening an issue on [Github](https://github.com/huggingface/huggingface_hub). rNdataTr)r/r0z HTTP Error z thrown while requesting r!'z' thrown while requesting z Retrying in z s [Retry /z].rr7) isinstancetypeintioIOBasertellrVseekr2 status_coder-warningraise_for_statusr;ConnectionErrorrLtimesleepmin)r/r0rZr[r\r]r^r3nb_tries sleep_timeio_obj_initial_posrHresponseerrr7r7r8 http_backoffsH G      ruendpointcCsD|r|dntj}|tjtjfvr |tj|}|tj|}|S)zReplace the default endpoint in a URL by a custom one. This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint. ra)rstriprENDPOINT_HF_DEFAULT_ENDPOINT_HF_DEFAULT_STAGING_ENDPOINTreplace)r0rvr7r7r8fix_hf_endpoint_in_urlPs r|rs endpoint_namec CsLz|WdSty%}z|jd}|jd}|dkr7|jddd|jd}tt||||dkrQ|jddd |jd}tt||||d krk|jddd |jd}tt ||||d kr|jddd |jddd }tt ||||dks|jdkr|j dur|j jdurt |j jdur|jddd|jdd}tt||||jdkr|durd|dnd}tt||||jdkrd|jd|dd|jdd}tt||||jdkr|j jd}|d|d|jd d}tt|||ttt|||d}~ww)!a Internal version of `response.raise_for_status()` that will refine a potential HTTPError. Raised exception will be an instance of `HfHubHTTPError`. This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub. Example: ```py import requests from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError response = get_session().post(...) try: hf_raise_for_status(response) except HfHubHTTPError as e: print(str(e)) # formatted message e.request_id, e.server_message # details returned by server # Complete the error message with additional information once it's raised e.append_to_message(" `create_commit` expects the repository to exist.") raise ``` Args: response (`Response`): Response from the server. endpoint_name (`str`, *optional*): Name of the endpoint that has been called. If provided, the error message will be more complete. Raises when the request has failed: - [`~utils.RepositoryNotFoundError`] If the repository to download from cannot be found. This may be because it doesn't exist, because `repo_type` is not set correctly, or because the repo is `private` and you do not have access. - [`~utils.GatedRepoError`] If the repository exists but is gated and the user is not on the authorized list. - [`~utils.RevisionNotFoundError`] If the repository exists but the revision couldn't be find. - [`~utils.EntryNotFoundError`] If the repository exists but the entry (e.g. the requested file) couldn't be find. - [`~utils.BadRequestError`] If request failed with a HTTP 400 BadRequest error. - [`~utils.HfHubHTTPError`] If request failed for a reason not listed above. z X-Error-CodeX-Error-MessageRevisionNotFoundz Client Error. zRevision Not Found for url: . EntryNotFoundzEntry Not Found for url: GatedRepoz!Cannot access gated repo for url z$Access to this resource is disabled.z!Cannot access repository for url  RepoNotFoundiNzRepository Not Found for url: z Please make sure you specified the correct `repo_id` and `repo_type`. If you are trying to access a private or gated repo, make sure you are authenticated.iz Bad request for z endpoint:z Bad request:iz Forbidden: z Cannot access content at: z2 Make sure your token has the correct permissions.iRangez. Requested range: z. Content-Range: z Content-Range)rkr r&r'rir0_formatrrrrr2REPO_API_REGEXsearchrrrr))rsr}r> error_code error_messagemessage range_headerr7r7r8hf_raise_for_status]sx8             r error_typecustom_messagecCsg}|jd}|dur||z8|}|d}|dur/t|tr*||n|||d}|durH|D] }d|vrG||dq:Wntyh|jdd}|jrfd| vrf||jYnwdd |D}tt |}d |} |} | r| | vrd |vr| d | 7} n| d | 7} t |jtd} | rd | d } nt |jtd} | rd| d } | r| | vrd | vr| d } | d| | | | d} n| | 7} || || pddS)Nr~errorerrorsrz Content-Typer htmlcSs$g|]}t|rt|qSr7)r)strip).0liner7r7r8 s$z_format..rrz (Request ID: r"z (Amzn Trace ID: )rsserver_message)r&r'appendjsonrblistextendrtextlowerdictfromkeysjoinr)r(r%indexr)rrrs server_errors from_headersr_rr content_typerfinal_error_messager?request_id_message newline_indexr7r7r8rsX             r)r9N)N)I__doc__rerRrerTrmr* functoolsrhttprtypingrrrrrr;r r requests.adaptersr requests.modelsr huggingface_hub.errorsr r rrrrrrrrrr_fixesr_lfsr_typingr get_loggerr@r-r%r(compileVERBOSErrrDrErIBACKEND_FACTORY_TrJ__annotations__rMrVrLrdrQTimeoutrlSERVICE_UNAVAILABLEr)float Exceptionrur|rrr7r7r7r8s      $      $     ~ "