返回模型

说明文档

启用键值缓存的 Vicuna 7B v1.5 ONNX fp16 格式

模型创建者：LMSYS
原始模型：LMSYS Vicuna 7B v1.5

描述

本仓库包含由 Esperanto Technologies 完成的 Vicuna 7B v1.5 ONNX 转换后的 ONNX 文件。该模型采用 fp16 格式，并启用了 KVC（键值缓存）。

如何下载 ONNX 模型和权重文件

获取模型最简单的方法是克隆整个仓库。或者，您可以使用 huggingface-hub Python 库下载文件。

pip3 install huggingface-hub>=0.17.1

然后，您可以使用如下命令将任何单个模型文件高速下载到当前目录：

huggingface-cli download Esperanto/vicuna-7b-v1.5-kvc-fp16-onnx --local-dir vicuna-7b-v1.5-kvc-fp16-onnx --local-dir-use-symlinks False

有关使用 huggingface-cli 下载的更多文档，请参阅：HF -> Hub Python 库 -> 下载文件 -> 从 CLI 下载。

如何使用 ONNXRuntime 从 Python 代码运行

此模型可以使用 ONNXRuntime 在 CPU 上轻松运行。

首先安装依赖包

pip3 install onnx==1.16.1
pip3 install onnxruntime==1.17.1

示例代码：使用此模型生成文本

我们使用贪婪解码定义循环：

import numpy as np
import onnxruntime
import onnx
from transformers import AutoTokenizer

def generate_text(model_path, prompt, tokenizer, max_gen_tokens, total_sequence, window, context):
    model = onnx.load(model_path)

    #we create the inputs for the first iteration
    input_tensor = tokenizer(prompt, return_tensors=\"pt\")
    prompt_size = len(input_tensor['input_ids'][0])
    actual_input = input_tensor['input_ids']
    if prompt_size < window:
        actual_input = np.concatenate((tokenizer.bos_token_id*np.ones([1, window - prompt_size], dtype = 'int64'),
                                       actual_input), axis=1)
    if prompt_size + max_gen_tokens > total_sequence:
        print(\"ERROR: Longer total sequence is needed!\")
        return
    first_attention = np.concatenate((np.zeros([1, total_sequence - window], dtype = 'int64'),
                                      np.ones((1, window), dtype = 'int64')), axis=1)
    max_gen_tokens += prompt_size #we need to generate on top of parsing the prompt
    inputs_names =[node.name for node in model.graph.input]
    output_names =[node.name for node in model.graph.output]
    n_heads = 32 #gqa-heads of the kvc
    inputs_dict = {}
    inputs_dict['input_ids'] = actual_input[:, :window].reshape(1, window).numpy()
    inputs_dict['attention_mask'] = first_attention
    index_pos = sum(first_attention[0])
    inputs_dict['position_ids'] = np.concatenate((np.zeros([1, total_sequence - index_pos], dtype = 'int64'), np.arange(index_pos, dtype = 'int64').reshape(1, index_pos)), axis=1)
    inputs_dict['tree_attention'] = np.triu(-65504*np.ones(total_sequence), k= 1).astype('float16').reshape(1, 1, total_sequence, total_sequence)
    for name in inputs_names:
        if name == 'input_ids' or name == 'attention_mask' or name == 'position_ids' or name == 'tree_attention': continue
        inputs_dict[name] = np.zeros([1, n_heads, context-window, 128], dtype=\"float16\")
    index = 0
    new_token = np.array([10])
    next_index = window
    old_j = 0
    total_input = actual_input.numpy()

    rt_session = onnxruntime.InferenceSession(model_path)
    ## We run the inferences
    while next_index < max_gen_tokens:
        if new_token.any() == tokenizer.eos_token_id:
            break
        #inference
        output = rt_session.run(output_names, inputs_dict)
        outs_dictionary = {name: content for (name, content) in zip (output_names, output)}
        #we prepare the inputs for the next inference
        for name in inputs_names:
            if name == 'input_ids':
                old_j = next_index
                if next_index < prompt_size:
                    if prompt_size - next_index >= window: next_index += window
                    else: next_index = prompt_size 
                    j = next_index - window
                else:
                    next_index +=1
                    j = next_index - window
                    new_token = outs_dictionary['logits'].argmax(-1).reshape(1, window)
                    total_input = np.concatenate((total_input, new_token[: , -1:]), axis = 1)
                inputs_dict['input_ids']= total_input[:, j:next_index].reshape(1, window)
            elif name == 'attention_mask':
                inputs_dict['attention_mask'] = np.concatenate((np.zeros((1, total_sequence-next_index), dtype = 'int64'), np.ones((1, next_index), dtype = 'int64')), axis=1)
            elif name == 'position_ids':
                inputs_dict['position_ids'] = np.concatenate((np.zeros([1, total_sequence - next_index], dtype = 'int64'), np.arange(next_index, dtype = 'int64').reshape(1, next_index)), axis=1)
            elif name == 'tree_attention': continue
            else:
                old_name = name.replace(\"past_key_values\", \"present\")
                inputs_dict[name] = outs_dictionary[old_name][:, :, next_index-old_j:context-window+(next_index - old_j), :]

    answer = tokenizer.decode(total_input[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)
    return answer

现在我们运行推理：

tokenizer = AutoTokenizer.from_pretrained(\"Esperanto/vicuna-7b-v1.5-kvc-fp16-onnx\")
model_path = \"vicuna-7b-v1.5-kvc-fp16-onnx/model.onnx\"

max_gen_tokens = 20    #number of tokens we want tog eneral
total_sequence = 128   #total sequence_length
context = 1024         #the context to extend the kvc
window = 16            #number of tokens we want to parse at the time
messages = [
    {\"role\": \"system\", \"content\": \"You are a pirate chatbot who always responds in pirate speak!\"},
    {\"role\": \"user\", \"content\": \"Who are you?\"},
]

prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)

generated = generate_text(model_path, prompt, tokenizer, max_gen_tokens, total_sequence, window, context)
print(generated)

Esperanto/vicuna-7b-v1.5-kvc-fp16-onnx

作者 Esperanto

↓ 0 ♥ 0

创建时间: 2024-08-02 11:31:43+00:00

更新时间: 2024-12-11 14:37:23+00:00

在 Hugging Face 上查看

文件 (299)

.gitattributes

README.md

config.json

model.embed_tokens.weight

model.layers.0.input_layernorm.weight

model.layers.0.post_attention_layernorm.weight

model.layers.1.input_layernorm.weight

model.layers.1.post_attention_layernorm.weight

model.layers.10.input_layernorm.weight

model.layers.10.post_attention_layernorm.weight

model.layers.11.input_layernorm.weight

model.layers.11.post_attention_layernorm.weight

model.layers.12.input_layernorm.weight

model.layers.12.post_attention_layernorm.weight

model.layers.13.input_layernorm.weight

model.layers.13.post_attention_layernorm.weight

model.layers.14.input_layernorm.weight

model.layers.14.post_attention_layernorm.weight

model.layers.15.input_layernorm.weight

model.layers.15.post_attention_layernorm.weight

model.layers.16.input_layernorm.weight

model.layers.16.post_attention_layernorm.weight

model.layers.17.input_layernorm.weight

model.layers.17.post_attention_layernorm.weight

model.layers.18.input_layernorm.weight

model.layers.18.post_attention_layernorm.weight

model.layers.19.input_layernorm.weight

model.layers.19.post_attention_layernorm.weight

model.layers.2.input_layernorm.weight

model.layers.2.post_attention_layernorm.weight

model.layers.20.input_layernorm.weight

model.layers.20.post_attention_layernorm.weight

model.layers.21.input_layernorm.weight

model.layers.21.post_attention_layernorm.weight

model.layers.22.input_layernorm.weight

model.layers.22.post_attention_layernorm.weight

model.layers.23.input_layernorm.weight

model.layers.23.post_attention_layernorm.weight

model.layers.24.input_layernorm.weight

model.layers.24.post_attention_layernorm.weight

model.layers.25.input_layernorm.weight

model.layers.25.post_attention_layernorm.weight

model.layers.26.input_layernorm.weight

model.layers.26.post_attention_layernorm.weight

model.layers.27.input_layernorm.weight

model.layers.27.post_attention_layernorm.weight

model.layers.28.input_layernorm.weight

model.layers.28.post_attention_layernorm.weight

model.layers.29.input_layernorm.weight

model.layers.29.post_attention_layernorm.weight

model.layers.3.input_layernorm.weight

model.layers.3.post_attention_layernorm.weight

model.layers.30.input_layernorm.weight

model.layers.30.post_attention_layernorm.weight

model.layers.31.input_layernorm.weight

model.layers.31.post_attention_layernorm.weight

model.layers.4.input_layernorm.weight

model.layers.4.post_attention_layernorm.weight

model.layers.5.input_layernorm.weight

model.layers.5.post_attention_layernorm.weight

model.layers.6.input_layernorm.weight

model.layers.6.post_attention_layernorm.weight

model.layers.7.input_layernorm.weight

model.layers.7.post_attention_layernorm.weight

model.layers.8.input_layernorm.weight

model.layers.8.post_attention_layernorm.weight

model.layers.9.input_layernorm.weight

model.layers.9.post_attention_layernorm.weight

model.norm.weight

model.onnx ONNX

onnx__MatMul_5288

onnx__MatMul_5289

onnx__MatMul_5290

onnx__MatMul_5316

onnx__MatMul_5318

onnx__MatMul_5319

onnx__MatMul_5320

onnx__MatMul_5322

onnx__MatMul_5323

onnx__MatMul_5324

onnx__MatMul_5350

onnx__MatMul_5352

onnx__MatMul_5353

onnx__MatMul_5354

onnx__MatMul_5356

onnx__MatMul_5357

onnx__MatMul_5358

onnx__MatMul_5384

onnx__MatMul_5386

onnx__MatMul_5387

onnx__MatMul_5388

onnx__MatMul_5390

onnx__MatMul_5391

onnx__MatMul_5392

onnx__MatMul_5418

onnx__MatMul_5420

onnx__MatMul_5421

onnx__MatMul_5422

onnx__MatMul_5424

onnx__MatMul_5425

onnx__MatMul_5426

onnx__MatMul_5452

onnx__MatMul_5454

onnx__MatMul_5455

onnx__MatMul_5456

onnx__MatMul_5458

onnx__MatMul_5459

onnx__MatMul_5460

onnx__MatMul_5486

onnx__MatMul_5488

onnx__MatMul_5489

onnx__MatMul_5490

onnx__MatMul_5492

onnx__MatMul_5493

onnx__MatMul_5494

onnx__MatMul_5520

onnx__MatMul_5522

onnx__MatMul_5523

onnx__MatMul_5524

onnx__MatMul_5526

onnx__MatMul_5527

onnx__MatMul_5528

onnx__MatMul_5554

onnx__MatMul_5556

onnx__MatMul_5557

onnx__MatMul_5558

onnx__MatMul_5560

onnx__MatMul_5561

onnx__MatMul_5562

onnx__MatMul_5588

onnx__MatMul_5590

onnx__MatMul_5591

onnx__MatMul_5592

onnx__MatMul_5594

onnx__MatMul_5595

onnx__MatMul_5596

onnx__MatMul_5622

onnx__MatMul_5624

onnx__MatMul_5625

onnx__MatMul_5626

onnx__MatMul_5628

onnx__MatMul_5629

onnx__MatMul_5630

onnx__MatMul_5656

onnx__MatMul_5658

onnx__MatMul_5659

onnx__MatMul_5660

onnx__MatMul_5662

onnx__MatMul_5663

onnx__MatMul_5664

onnx__MatMul_5690

onnx__MatMul_5692

onnx__MatMul_5693

onnx__MatMul_5694

onnx__MatMul_5696

onnx__MatMul_5697

onnx__MatMul_5698

onnx__MatMul_5724

onnx__MatMul_5726

onnx__MatMul_5727

onnx__MatMul_5728

onnx__MatMul_5730

onnx__MatMul_5731

onnx__MatMul_5732

onnx__MatMul_5758

onnx__MatMul_5760

onnx__MatMul_5761

onnx__MatMul_5762

onnx__MatMul_5764

onnx__MatMul_5765

onnx__MatMul_5766

onnx__MatMul_5792

onnx__MatMul_5794

onnx__MatMul_5795

onnx__MatMul_5796

onnx__MatMul_5798

onnx__MatMul_5799

onnx__MatMul_5800

onnx__MatMul_5826

onnx__MatMul_5828

onnx__MatMul_5829

onnx__MatMul_5830

onnx__MatMul_5832

onnx__MatMul_5833

onnx__MatMul_5834

onnx__MatMul_5860

onnx__MatMul_5862

onnx__MatMul_5863

onnx__MatMul_5864

onnx__MatMul_5866

onnx__MatMul_5867

onnx__MatMul_5868

onnx__MatMul_5894

onnx__MatMul_5896

onnx__MatMul_5897

onnx__MatMul_5898

onnx__MatMul_5900

onnx__MatMul_5901

onnx__MatMul_5902

onnx__MatMul_5928

onnx__MatMul_5930

onnx__MatMul_5931

onnx__MatMul_5932

onnx__MatMul_5934

onnx__MatMul_5935

onnx__MatMul_5936

onnx__MatMul_5962

onnx__MatMul_5964

onnx__MatMul_5965

onnx__MatMul_5966

onnx__MatMul_5968

onnx__MatMul_5969

onnx__MatMul_5970

onnx__MatMul_5996

onnx__MatMul_5998

onnx__MatMul_5999

onnx__MatMul_6000

onnx__MatMul_6002

onnx__MatMul_6003

onnx__MatMul_6004

onnx__MatMul_6030

onnx__MatMul_6032

onnx__MatMul_6033

onnx__MatMul_6034

onnx__MatMul_6036

onnx__MatMul_6037

onnx__MatMul_6038

onnx__MatMul_6064

onnx__MatMul_6066

onnx__MatMul_6067

onnx__MatMul_6068

onnx__MatMul_6070

onnx__MatMul_6071

onnx__MatMul_6072

onnx__MatMul_6098

onnx__MatMul_6100

onnx__MatMul_6101

onnx__MatMul_6102

onnx__MatMul_6104

onnx__MatMul_6105

onnx__MatMul_6106

onnx__MatMul_6132

onnx__MatMul_6134

onnx__MatMul_6135

onnx__MatMul_6136

onnx__MatMul_6138

onnx__MatMul_6139

onnx__MatMul_6140

onnx__MatMul_6166

onnx__MatMul_6168

onnx__MatMul_6169

onnx__MatMul_6170

onnx__MatMul_6172

onnx__MatMul_6173

onnx__MatMul_6174

onnx__MatMul_6200

onnx__MatMul_6202

onnx__MatMul_6203

onnx__MatMul_6204

onnx__MatMul_6206

onnx__MatMul_6207

onnx__MatMul_6208

onnx__MatMul_6234

onnx__MatMul_6236

onnx__MatMul_6237

onnx__MatMul_6238

onnx__MatMul_6240

onnx__MatMul_6241

onnx__MatMul_6242

onnx__MatMul_6268

onnx__MatMul_6270

onnx__MatMul_6271

onnx__MatMul_6272

onnx__MatMul_6274

onnx__MatMul_6275

onnx__MatMul_6276

onnx__MatMul_6302

onnx__MatMul_6304

onnx__MatMul_6305

onnx__MatMul_6306

onnx__MatMul_6308

onnx__MatMul_6309

onnx__MatMul_6310

onnx__MatMul_6336

onnx__MatMul_6338

onnx__MatMul_6339

onnx__MatMul_6340

onnx__MatMul_6342

onnx__MatMul_6343

onnx__MatMul_6344

onnx__MatMul_6370

onnx__MatMul_6372

onnx__MatMul_6373

onnx__MatMul_6374

onnx__MatMul_6376

special_tokens_map.json

tokenizer.json

tokenizer.model

tokenizer_config.json