o gd#@s8ddlZddlZddlZddlmZmZddlZddlmZ ddl Z ddl m Z m Z mZmZddlmZddlmZddlmZeeZGdd d Zd d Zed kreZejr`eejejZ ej!Z"ej#$e"r}e%d e"de&d e"de 'e Z(ee(ej)ej*ej+dZ,e,-e,j(.e"ddSdS)N)ListTuple) GraphProto ModelProto NodeProto TensorProto)quantize_matmul_bnb4) ONNXModel)attribute_to_kwargc@seZdZdZdZdZddededefdd Ze d e e d e e e ffd d Zdejd ejfddZdede e d efddZde e fddZddZdS)MatMulBnb4QuantizerzMPerform 4b quantization of constant MatMul weights using FP4 or NF4 data typerr Nmodel quant_type block_sizecCs@|pg}|tjtjfvsJt||_||_||_t||_dSN) r FP4NF4r r rrsetnodes_to_exclude)selfr rrrre/var/www/visachat/venv/lib/python3.10/site-packages/onnxruntime/quantization/matmul_bnb4_quantizer.py__init__&s  zMatMulBnb4Quantizer.__init__ graph_pathreturncCsLtt|dddD]}||}|jD]}|j|kr"||fSqq dS)Nr )NN)rangelen initializername)rrgidgraphtensorrrr__get_initializer.s  z%MatMulBnb4Quantizer.__get_initializerfpweightc Cst|jdkr td|}|j\}}||}|j}||d|}|dd}tj|dd} tj||jd} t | || ||j ||| | fS)z4b quantize fp32/fp16 weightz9Current bnb4 block quantization only supports 2D tensors!r uint8)dtype) rshape ValueError transposecopyrnpzerosr'rr) rr$ fpweight_trowscolsnumelr num_blocksquantized_numelpackedabsmaxrrrbnb4_block_quant7s   z$MatMulBnb4Quantizer.bnb4_block_quantnode graph_stackcCs|jdkr|Std|jd|j|jvr#td|jd|S|jd}t||\}}|dur;td|Stj |}t |j d krOtd |S| |\}}tj |} |jd | _|jD]} | j|krt|j| nqetj |} |jd | _|j| | gi} |j \} }| | d <|| d<|j| d<|j| d<tjj d|jd| j| jg|jdg|jr|jd nddd| }td|jd|S)zdIf the node is MatMul with fp32 const weight, quantize the weight with int4, and return the new nodeMatMulzstart to quantize z ...zexclude to quantize z$ as specified by nodes_to_exclude...r Nz2MatMul doesn't have const weight. Skip to quantizer%z)MatMul weight is not 2D. Skip to quantize_Bnb4_absmaxKNrr MatMulBnb4r com.microsoft)inputsoutputsrdomainzcomplete quantization of )r>)op_typeloggerdebugrrinputr %_MatMulBnb4Quantizer__get_initializeronnx numpy_helperto_arrayrr(r6 from_arrayremoverextendrrhelper make_nodeoutput)rr7r8inputBBBs_graphB_arrayr4r5B_quantrG absmax_tensorkwargsr/r0matmul_bnb4_noderrr_bnb4_matmul_node_weightMsX                  z,MatMulBnb4Quantizer._bnb4_matmul_node_weightc Csg}|d}|jD]s}dd|jD}t|rsi}|jD]D}|jtjjkr4||j|j | |i}n'|jtjj krWg} |j D]} || | | |gq@|j | i}nt|}||qtjj|j|j|jfd|j i|}||||q |d|j |||S)NrcSs,g|]}|jtjjks|jtjjkr|qSr)typerIAttributeProtoGRAPHGRAPHS).0attrrrr s z9MatMulBnb4Quantizer._process_subgraph..rr7)r7 attributerr[rIr\r]appendgr_process_subgraphr^graphsrNr updaterOrPrDrGrQrZ ClearFieldpop) rr8 new_nodesr!r7 graph_attrsrXr`kvvaluesubgraphrrrres@          z%MatMulBnb4Quantizer._process_subgraphcCsd|jg}|j}d}|D] }|jdkrd}q|s&|tjddg|||j dS)NFr@Tr ) r r! opset_importrCrNrIrO make_opsetidreclean_initializers)rr8ro has_ms_domainopsetrrrprocesss    zMatMulBnb4Quantizer.processr)__name__ __module__ __qualname____doc__rrrintr staticmethodrrrrrHnpt ArrayLiker,ndarrayr6rrZrertrrrrr s 7 &r cCstjdd}|jdddd|jdddd|jd d d tjtjgd d |jdd ddd|jddd dd|jd d|jddtd gdd|S)NaBlockwise FP4/NF4 quantization for MatMul 2D weight matrices. A weight matrix is partitioned into blocks, where each block is a contiguous subset inside the flattened transposed weight matrix. Each block is quantized into a set of 4b integers with an absolute value scaling factor. ) descriptionz --input_modelTzPath to the input model file)requiredhelpz--output_modelzPath to the output model filez --quant_typeFr z&Quantization data type. 0: FP4, 1: NF4)rdefaultchoicesrz --block_size@zVBlock size for blockwise quantization. Note: bnb.nn.Linear4bit only uses block_size=64)rrrz-vz --verbose store_true)raction)verbosez--nodes_to_exclude+zBSpecify the nodes to be excluded from quantization with node names)nargsr[rrr) argparseArgumentParser add_argumentr rr set_defaultsstr parse_args)parserrrrrs:    r__main__zfile z already exists)rT)/rloggingostypingrrnumpyr, numpy.typingr{rI onnx.onnx_pbrrrronnxruntime.capi._pybind_stater onnx_modelr quant_utilsr getLoggerrurEr rargsrsetLevelDEBUG input_modelinput_model_path output_modeloutput_model_pathpathexistserror Exceptionloadr rrrquantrtsave_model_to_filerrrrs:     "'