o g_@sddlZddlZddlmZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZmZmZmZmZdd lmZmZmZd ejd <eeZd ej iZ!dCd dZ"ddZ#ddZ$ej%fddZ&ddZ'ddZ(ddZ)ddZ* dDddZ+de,d e,d!e-d"e.d#e.d$e d%e.d&e.fd'd(Z/d)e,d*e,d+e,fd,d-Z0d.d/Z1 dDd0d1Z2d2d3Z3dEd5d6Z4d7d8Z5d9d:Z6d;d<Z7d=d>Z8d?d@Z9dAdBZ:dS)FN)Path)AffinitySetting) OptimizerInfo Precisioncreate_onnxruntime_session) MODEL_CLASSES)QuantizeHelper)torch_onnx_export) AutoConfigAutoFeatureExtractor AutoTokenizer LxmertConfigTransfoXLConfig)PRETRAINED_GPT2_MODELSGPT2ModelNoPastStateTFGPT2ModelNoPastState2TF_CPP_MIN_LOG_LEVELtriucCs|dusJt|jdkr|d|dksJtd}|tjdtjd|}|d|dd|df}t||t |S)Nrr)rdtype) lenshapesize torch_functorchonesuint8wherebool zeros_like)xdiagonalout torch_triutemplatemaskr*]/var/www/visachat/venv/lib/python3.10/site-packages/onnxruntime/transformers/onnx_exporter.py triu_onnx#s & r,cCs tt_dSN)r,rrr*r*r*r+replace_torch_functions-s r.cCstdt_dS)Nr)rrrr*r*r*r+restore_torch_functions1sr/c Cs|jdvrtj|d|j|jtj}d|i}|Stjjd|d||f|d}d|i}d|vr=tj||g|d }||d<d |vrNtj ||g|d } | |d <|j rU||d <t |t rxtj dd|jtj|d <tj dd|jtj|d <t |trtj |jgtjd |d<|S)Nvitswin pixel_valuesrrlowhighrr input_idsattention_maskrtoken_type_idsdecoder_input_ids visual_feats visual_posz@tf_transfo_xl_model/transformer/pos_emb/einsum/Einsum/inputs_1:0) model_typenumpyrandomrand image_sizeastypefloat32randintrzerosis_encoder_decoder isinstancer randnvisual_feat_dimvisual_pos_dimr hidden_size) vocab_size batch_sizesequence_length input_namesconfig data_typer8inputsr9 segment_idsr*r*r+create_onnxruntime_input5s,     rUcCs&i}|D] }||vr||||<q|Sr-r*)rSrPremaining_model_inputs input_namer*r*r+ filter_inputsSs  rXcCs$t|ttfrdd|DgS|gS)NcSsg|]}t|qSr*)flatten.0ir*r*r+ \szflatten..)rHlisttuple)rSr*r*r+rY[s$rYcCs0|D]}t|ttfs||nt||q|Sr-)rHr^r_appendupdate_flatten_list)rSres_listr\r*r*r+ra_s rac Cs|djd}dd|D}ddtt|D}t|D]%\}}ddi||<||j}t|D]\}} | |krA|||d iq0q||fS) Nr8cSsi|]}|dddqS)rNseq_len)rrr*r[keyr*r*r+ hsz&build_dynamic_axes..cSsg|] }dt|dqS)output_r)strrZr*r*r+r]jsz&build_dynamic_axes..rrNrd)rranger enumerateupdate) example_inputsoutputs_flattenrO dynamic_axes output_namesr\ output_namedimsjdimr*r*r+build_dynamic_axeses  ruc CsNt||dd}|durt|ddSt|ddd|D}|||}t|t|krEtdt|dt|dStt|D]Q} t t || ||  } | d krntd | d | |rrd nd } |rxd nd } tj || ||  | | dstd| d| d| dSqKtd|dS)NF)enable_all_optimizationz is an invalid ONNX modelz is a valid ONNX modelcSsi|] \}}||qSr*)r?)r[ktr*r*r+rgsz'validate_onnx_model..z"Number of output tensors expected z, got g-C6?zMax absolute diff=z for output tensor g?g?)rtolatolzOutput tensor z is not close: rtol=z, atol=z0inference result of onnxruntime is validated on T) rloggererrorinfoitemsrunrrjr?amaxabscpuallclose) onnx_model_pathrmexample_outputs_flattenuse_gpufp16rp test_sessionexample_ort_inputsexample_ort_outputsr\abs_diffryrzr*r*r+validate_onnx_modelts: $   ronnx_dir model_name input_countoptimized_by_scriptr precisionoptimized_by_onnxruntimeuse_external_datac Csddlm}|dd|} |s| d|} n|rdnd} | d|d|d| } |r/| d7} |} |rG|sGtj|| } tj| sGt| tj| | dS) Nr)subz [^a-zA-Z0-9_]_gpur_ortz.onnx)rerospathjoinexistsmakedirs) rrrrrrrrrnormalized_model_namefilenamedevice directoryr*r*r+get_onnx_file_paths    r file_pathsuffixreturncCs&t|}t|j|j||jS)a Append a suffix at the filename (before the extension). Args: path: pathlib.Path The actual path object we would like to add a suffix suffix: The suffix to add Returns: path with suffix appended at the end of the filename and before extension )rriparentjoinpathstem with_suffixr)rrrr*r*r+add_filename_suffixsrcCsh|stj|s*t|jjdddddlm}m}||||dd}||||<dSt d|dS)NTparentsexist_okr)get_fusion_statisticsoptimize_by_onnxruntimec)roptimized_model_path opt_level'Skip optimization since model existed: ) rrrrrmkdir optimizerrrr{r})rort_model_pathr overwritemodel_fusion_statisticsrrrr*r*r+optimize_onnx_model_by_ortsrc  Cs|stj|stt|jjdddddlm} ddlm } | dur&| |} | ||t j kr3d| _ |t jkr;d| _|dkrCd}d}| ||||d| |dd}|d ksW|d kr[||| |<|t j krl|jdd ||| dStd |dS) NTrr) FusionOptions)optimize_modelFr2) num_headsrLroptimization_optionsronly_onnxruntime bert_kerasbert_tf) keep_io_typesr)rrrrrrfusion_optionsrrruse_raw_attention_maskrFLOAT16enable_gelu_approximationINT8enable_embed_layer_normuse_dynamic_axesget_fused_operator_statisticsconvert_float_to_float16save_model_to_filer{r})rrr>num_attention_headsrLrrrrruse_external_data_formatrrr opt_modelr*r*r+optimize_onnx_models>         rcCsz|dur|tvr |Stddt|tvrdSddl}|d|dur'dS|d|dur1dS|d |dur;d Sd S) NzValid model class:  rrz-squad$AutoModelForQuestionAnsweringz-mprc$"AutoModelForSequenceClassificationgpt2AutoModelWithLMHead AutoModel)r Exceptionrrrsearch)rcustom_model_classrr*r*r+modelclass_dispatchersrFcCszt||}|dkr|rtj|||dStj|||dS|r!d|}td|gd}td|t||}|j|||dS)Nr)rQ cache_dirTF transformers)fromlistzModel class name: )rrfrom_pretrainedr __import__r{r}getattr)rrQrr is_tf_modelmodel_class_nametransformers_module model_classr*r*r+load_pretrained_model/s  rcCs@tj||d}t|drd|_||t||||d}||fS)Nr return_dictF)rQrr)r rhasattrrmodifyr)rrrconfig_modifierrQmodelr*r*r+ load_pt_modelBs   rcCsHtj||d}||t}|t||||dd}|||fS)NrT)rQrrr)r rrr get_affinityr set_affinity)rrrrrQaffinity_settingrr*r*r+ load_tf_modelNs rcCs ddlm}||\}}||fS)Nr)tf2pt_pipeline)convert_tf_models_to_pytorchr)rrrQrr*r*r+load_pt_model_from_tfcs  rc Cs&d}|r t| |||d|}|tjkr| || jfS|tjks'|tjks'|tjkrqt||t |d||d|}t | ||| j | j ||| | | || |} |rUt| ||||tjk|}|tjkrqt d| t| | |t d| |tjkr|rt| d}t| ||| | | ||dvr| jfS| jfS)NTFzQuantizing model: zFinished quantizing model: rr0)rrNOOPTrMBYSCRIPTrrrrrrrrLr{r}rquantize_onnx_modelBYORTrr num_labels)rrr>rrPrr optimize_info validate_onnxrrrQrrrmrrpris_valid_onnx_modelrrr*r*r+validate_and_optimize_onnxms        rc Cst||||\}}|d}d}|dvr;tj||d}tjjdd|j|jdtjd |j|jd}||dd}nt j||d}|j |d }|j d dd}t||}|di|}t|ttfslJd t|t|}t|g}t||t|d | | d |}|stj|std |t|jjdddd}d}|dvrdd|Ddg}}nt||\}}t t!|t|"|t|#||d||d t$ntd|t%|||||| | | | | ||||||d|\}}}||||fS)Nr0rrr3r5pt)return_tensorsrThis is a sample inputz%type of output is not list or tuple: FExporting ONNX model to TrcSsi|]}|ddiqS)rr4r*rer*r*r+rgsz-export_onnx_model_from_pt..logits) rargsfrPrprodo_constant_folding opset_versionr!Skip export since model existed: r*)&rrr rr?r@rErBr reshaper max_model_input_sizesget encode_plusrXrHr^r_typerYrarrrrrr{r}rrrrur.r valueskeysr/r) rrrr>rrrrrPrroptimizer_inforrrrrrQrrmmax_input_sizeimage_processordata tokenizerexample_outputsrrrorponnx_model_filerrMr*r*r+export_onnx_model_from_pts        rc(Csddl}|jgdtj||d}|jdur|ddi|j|d}t ||||\}}| t ||j dd|d d d }t ||}|jrT|j dd|d d d j|d <|d krp|jdd|jg|d<|jdd|jg|d<z|jrwd|_Wn tyYnw||dd}d}|dks|dkrdg}|d}ddlm}||}t||t |d| | d|}|r|ddn|}|stj|sYtd||st|j j!d d dddl"}ddl#}|j$%|j$j&g}|'D]\} }!dgt |!j(}"|)|j*t+|"|!j,| dq|j-j.|t+||||d\}#}#|rX|/|d}$|$0tj1|Wdn 1s6wYtj2tj1|d}tj|rRt3|t4||ntd||d}t5|||||| | | | | ||||||||\}%}&}'|%|&|'|fS) NrGPUr pad_tokenz[PAD]rrtf max_lengthT)rrpadding truncationr;zunc-nlp/lxmert-base-uncasedrr<r=F)trainingzxlnet-base-casedzxlnet-large-casedlast_hidden_state)nestrr)name)input_signatureopset large_model output_pathrz__MODEL_PROTO.onnxr_tf)6 tensorflowrQset_visible_devicesr rradd_special_tokensr r rresize_token_embeddingsrr rXrGr8r@normalrJrK use_cachertensorflow.python.utilrrYrrrrr{r}rrrzipfiletf2onnxlogging set_levelERRORr~rr` TensorSpecr_rconvert from_kerasZipFile extractalldirnamerremoverenamer)(rrrr>rrrrrPrrrrrrrrrrrrQrrmrrprrrtf_internal_model_pathr/r0specsr!valuerrrzoptimized_onnx_pathrrMr*r*r+export_onnx_model_from_tf;s           rA)rNr-)F);r1rpathlibrr?raffinity_helperrbenchmark_helperrrrhuggingface_modelsrquantize_helperrtorch_onnx_export_helperr rr r r r r0onnxruntime.transformers.models.gpt2.gpt2_helperrrrenviron getLogger__name__r{rrr,r.r/int64rUrXrYrarurriintr"rrrrrrrrrrrrAr*r*r*r+sl           , !  ;   _ o