o gj@sddlZddlmZmZddlZddlZddlZzddlm Z Wn e y+dZ Ynwddl m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZddlmZGd d d ZGd d d ZdS) N)AnyDict)to_array_extended) TensorData) ONNXModel) ONNX_TYPE_TO_NP_TYPETENSOR_NAME_QUANT_SUFFIX find_by_namemodel_has_infer_metadatanormalize_axispack_bytes_to_4bit quantize_dataquantize_nparray&save_and_reload_model_with_shape_infertensor_proto_to_array)TensorQuantOverridesHelperc@sLeZdZdeeeffddZdddZddZd d Z d d Z d dZ dS)QuantizationParamsdatacKsi|_|D]d\}}t|tstdt|d|d|dkr7t|tttjfs7tdt|d|d|dkrNt|tsN|durNtdt|d|dkrf|j tj tj fvrft d|j d|||j|<qdS) NzKeys must be strings not z for k=.axisz1Values must be numpy arrays, int, float, str not z'Axis value must be an int or None, not scalez5scale must a float32 or float16 numpy element but is ) ritems isinstancestr TypeErrortypeintnpndarraydtypefloat32float16 ValueError)selfrkvr'^/var/www/visachat/venv/lib/python3.10/site-packages/onnxruntime/quantization/base_quantizer.py__init__%s  zQuantizationParams.__init__NcCs|j||SN)rget)r$key default_valuer'r'r(r+2zQuantizationParams.getccs|jEdHdSr*rr$r'r'r(__iter__5szQuantizationParams.__iter__cCs |j|Sr*r/)r$r,r'r'r( __getitem__8 zQuantizationParams.__getitem__cCs||j|<dSr*r/)r$r,valuer'r'r( __setitem__;r.zQuantizationParams.__setitem__cCs t|jSr*)lenrr0r'r'r(__len__>r3zQuantizationParams.__len__r*) __name__ __module__ __qualname__rrrr)r+r1r2r5r7r'r'r'r(r$s  rc@seZdZ dddZdejjdefddZdd Z d d Z d d Z ddZ ddZ ddZd ddZd!ddZ  d"ddZddZdS)# BaseQuantizerNc Cst|st|}dd|jjD|_|jdd|jjD|jdd|jjDt||_ ||_ ||_ | r;| ni|_ d|j voG|j d|_ d|_d|j voU|j d|_|j dd|_|j dd |_|j d |_t|d ||_t|d ||_ |durttd d |rtdtdd|Dd||_||_||_| |_| |_!t"|j di|_#dd|j $D|_%|j#&|j%|j'|\} } | st(| |j#)|_*dS)NcSi|]}|j|qSr'name).0vir'r'r( Rz*BaseQuantizer.__init__..cSr<r'r=)r?otr'r'r(rASrBcSr<r'r=)r?itr'r'r(rATrBEnableSubgraphForceQuantizeNoInputCheckWeightSymmetricActivationSymmetricFMinimumRealRange tensor_typecSs t|t Sr*)rr)tr'r'r(ws z(BaseQuantizer.__init__..z(tensors_range contains unexpected types css|]}t|VqdSr*)r)r?r&r'r'r( ysz)BaseQuantizer.__init__..z, not TensorData.TensorQuantOverridescSr<r'r=)r?initzerr'r'r(rArB)+r rgraph value_info value_infosupdateoutputinputrmodel per_channel reduce_range extra_optionsenable_subgraph_quantizationparentforce_quantize_no_input_checkr+_is_weight_symmetricis_activation_symmetricmin_real_rangegetattractivation_qType weight_qTypeanymapvaluesrset tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizecheck_opset_version opset_versionrtensor_quant_overrides initializer initializersis_validkeysr#get_quant_typestensor_quant_override_qtypes) r$rVrWrXrbrargrhrirjrYoverrides_valid overrides_errr'r'r(r)CsJ    zBaseQuantizer.__init__weight_quant_typereturncCs0|jdur|jS|tjjtjjtjjtjjfvSr*)r]onnx TensorProtoINT4INT8INT16 FLOAT8E4M3FN)r$rvr'r'r(is_weight_symmetrics z!BaseQuantizer.is_weight_symmetriccCstr*)NotImplementedErrorr0r'r'r(quantize_modelszBaseQuantizer.quantize_modelcCst||j}|duSr*)r rVrn)r$ input_namernr'r'r(is_input_a_initializersz$BaseQuantizer.is_input_a_initializercCs|jSr*)rWr0r'r'r(is_per_channelszBaseQuantizer.is_per_channelcCsNt||j}|dur|jtjjtjjfvS|jr|j dur!dS|j |S)NF) r rVrn data_typerxryFLOATFLOAT16rZr[is_valid_quantize_weight)r$ weight_nameweightr'r'r(rs  z&BaseQuantizer.is_valid_quantize_weightcCsV|jdurt|jdkr|j|jvrdS|j|jvrdS|jdur)|j|jvr)dSdS)NrFT)rhr6r>op_typerjri)r$noder'r'r(should_quantize_nodes   z"BaseQuantizer.should_quantize_nodecCsdd|jjjD}t|dkrtd|dj}|dkr(td|ddS|dkrOtd|d |jjj|d|jjjt j d d gd }|d kr|j t j jkrtd|d |jjj|d|jjjt j d d gd|jj_d }|S)NcSs g|] }|jr |jdkr|qS)zai.onnx)domain)r?opsetr'r'r( sz5BaseQuantizer.check_opset_version..rz$Failed to find proper ai.onnx domainr z$The original model opset version is ze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z, which does not support quantization. Please update the model to opset >= 11. Updating the model automatically to opset 11. Please verify the quantized model. z, which does not support quantization to float 8. Please update the model to opset >= 19. Updating the model automatically to opset 19. Please verify the quantized model. )rV opset_importr6r#versionloggingwarningremoveextendrxhelper make_opsetidrbryr} ir_version)r$ai_onnx_domainrlr'r'r(rks6      z!BaseQuantizer.check_opset_version?cCst||j}t|}|t}|jtjjkr`t |}|j t j kr'tjj } n|j t jkr2tjj} n td|j d|t j} t jdg| j d} | d} tj| |} |j| gd}nz|||} t j |t jdt j | t jd} | } t t t jj}t t t jj}t | |kst | |krtd|dt | ||t j} t j | t jd|j!}tj||} |j| gt j | |j dd} d }|j} |d }tj| |}|j|g|jtjjkr|j}ntjj"}|d }|jtjjkrtj#$||jdgd g}n$| j%dkr.t j&| j't jdd}tj||}n tj#$||gd g}|j|g|||| || fS)z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale zEOnly float16 or float32 are supported with float 8 but bias dtype is rrr CastzQuantized bias `z<` exceeds the range of a int32. The bias scale is too small.DequantizeLinear_scale _zero_pointr)(r rVrnrr rbrxryr}rasarrayr r"rr!rrastypearrayreshape numpy_helper from_arrayinitializer_extendfloat64roundiinfoint32minmaxrcrrclipdimsINT32r make_tensorsizezerosshape)r$ bias_name input_scale weight_scalebetabias_initializer bias_dataquantized_bias_namer node_qtypequantized_data bias_scalebias_scale_datapacked_bias_initializer node_type int32_min int32_max bias_np_dataquantized_bias_scale_namepacked_bias_scale_initializerrJquantized_bias_zp_namepacked_bias_zp_initializer bias_zp_datar'r'r(quantize_bias_static_implsj           z'BaseQuantizer.quantize_bias_static_implFc Cs~|jt}|jd}|jd}t|}|jj|jid} d| vr%| dj}d| vr{d| vr{tj| dt|d} t| d} t || | | } t | tj sWJdt | | jtjkrc| jtjkskJd | jt | tj szJdt | na||jkr||n|j} t| || d | | d |jo||j| d | d d\} } } t | tj sJdt | | jtjkr| jtjksJd | jt | tj sJdt | |j}tj||g| d}tj||g| d}|j||g|s|jtj j!krkt }|j|_|j"#|j"||_| $%|_&t'durjt'|}|j(|j(ksB|%| %krjt)d|j(d| %ddd|%ddd|j(dt*|ddd nH|tj j+tj j,fvr| jtj-tj.fvrt)d|dt/t0| %}tjj|||j"|dd}ntj1| tj2|d|j"} tj34| |}|j|g|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name rr default_val quant_typer zero_pointrUnexpected type Unexpected dtype symmetricrXrminrmaxrXr_ rmin_override rmax_override)rNzThe initializer of shape z! could not be created, expecting rz, got z and shape=z raw=rQuantized weights for . must be 8-bit before packing as 4-bit values.Traw)5r>r rrmget_per_tensor_overridesrJrrrrflattenrrrr r!r"rbr~r^rr+rXr_rrxrrrtolistrVrryr}rrcopytobytesraw_datarr RuntimeErrorrrzUINT4int8uint8bytesr rtensor_dtype_to_np_dtyperr)r$rqTyperXkeep_float_weight q_weight_namezp_name scale_name weight_dataquant_overridesrr q_weight_datar scale_dtypescale_initializerzero_initializerq_weight_initializercheck packed_datar'r'r(quantize_initializer_impl1s              z'BaseQuantizer.quantize_initializer_implTc& Cst||j}|durtd|t|}t|j}t||\} } | s0td|d|d|| }|j|} |jj |d|igd} t| } | dkrY| | krYtd|d | d t| d d|\}}|rj||kr|td |d |d| d ddd| d vr| d dj }| d  d| |}| d  d|j o|}g}g}g}t|j}t|}d||<t| D]}|||}|| kr|nd }| |}d|vr5d|vr5tj|dt|d}t|d}t||||}t|tjsJdt||jtjkr |jtjksJd|jt|tjs$Jdt|t|tjs4Jdt|n]t|||||j| d| dd\}}}t|tjs\Jdt||jtjkrj|jtjksrJd|jt|tjsJdt|t|tjsJdt||||||t| |qt!||}|t"}|d}|d} |j#|g}!t$j%&| |j'|!t(|)}"t$j%&|||!t(|)}#|j*|"|#g|s=|t$j+j,t$j+j-fvr |jtj.tj/fvrt0d|dt1t2|3}$t$j%j&||||$dd }%|j*|%gntj|t$j%4|d |j#}t$j56||}%|j*|%g||| fS)!Nz{} is not an initializerzWeight z# has a per-channel axis with value z that is out-of-bounds for rank rrrz.Per-channel tensor quantization overrides for z must have either 1 or z& elements in the list of dictionaries.rz"Tensor quantization overrides for z& specify an unexpected axis. Expected z , but got rrrrXrrrrrrrrrrrrTr)7r rVrnr#rr6rr rmget_per_channel_overridesrJr+r~rXlistrangetakerrrrrrrrr r!r"rr_appendrr concatenater rrxrrrhstackrrryrzrrrrrr rrrr)&r$rrb channel_axisrXrrnweights weights_rank is_axis_valid axis_norm channel_countquant_overrides_for_channelsnum_channel_overridesis_axis_override_valid axis_overriderzero_point_list scale_listquantized_per_channel_data_list weights_shape reshape_dimsiper_channel_datachannel_override_indexchannel_quant_overridesrrquantized_per_channel_dataquantized_weightsrrrzero_scale_shaperrrrr'r'r( quantize_weight_per_channel_impls                           z.BaseQuantizer.quantize_weight_per_channel_implcCs|jdurdS|jD]p}|jdvr_||sq t|j|jddkr(q |jd|jvs8|jd|jvr9q |j|jd}t |t sVt dt |d|jdd||j|jd<q |jdkr|||sjq t t dt d d |j|jd<q dS) N)ClipRelurrrz for rSoftmaxrr)lowesthighest)rgrVnodesrrr6input_name_to_nodesrUrTrrrrrr!)r$rtdr'r'r(adjust_tensor_rangess(        $z"BaseQuantizer.adjust_tensor_rangesr*)r)FF)TF)r8r9r:r)rxryDataTypeboolr~rrrrrrkrrrr#r'r'r'r(r;Bs" K  # T`  r;)rtypingrrnumpyrrxonnx.numpy_helperonnx.reference.op_runr ImportError calibrater onnx_modelr quant_utilsrr r r r r rrrrrmrrr;r'r'r'r(s    0