目录

Quantization Effect Evaluation

To evaluate the effect of the quantized model, you can insert quantization operators into the floating-point model for inference by using to the plug-in method. The model inference framework and input remain unchanged.

Plug-in method:

The following is an example of inserting quantization operators into the floating-point model during inference to convert floating-point numbers to the quantized model:

import sys
import torch

sys.path.append("path/to/dopt_torch_py3")

def get_quanted_model(base_model, dopt_config, quanted_ckpt):
    from dopt.dopt_lm.opt_main import (optimize_model, set_quant_state, set_calibrate_state, set_run_mode,)
    model = optimize_model(base_model, dopt_config)
    model.load_state_dict(torch.load(quanted_ckpt, map_location=torch.device('cpu')), strict=True)
    set_quant_state(model, weight_state=True, input_state=True)
    set_calibrate_state(model, False)
    model.eval()
    return model

The parameters are defined as follows:

Complete inference example using the plug-in method:

The following provides a complete example of Qwen2 quantization inference for your reference. In this example, the plug-in method is added to verify the quantization effect. The quantization toolkit and quantization file are used as the input for simulation inference. If the inference result is normal, the quantization is successful.

import os
import sys
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# Enter the actual path of **tools_dopt/dopt_pytorch_py3** in the **DDK_tools** package.
sys.path.append('path/to.dopt')
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

def get_quanted_model(base_model, dopt_config, quanted_ckpt):
    from dopt.dopt_lm.opt_main import (optimize_model, set_quant_state, set_calibrate_state, set_run_mode,)
    model = optimize_model(base_model, dopt_config)
    model.load_state_dict(torch.load(quanted_ckpt, map_location=torch.device('cpu')), strict=True)
    set_quant_state(model, weight_state=True, input_state=True)
    set_calibrate_state(model, False)
    model.eval()
    return model

def generate(prompt="Give me a short introduction to large language model."):
    messages = [
        {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"},
        {"role": "user", "content": prompt}
    ]
    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
    print(model_inputs)
    generated_ids = model.generate(**model_inputs, max_new_tokens=512)
    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
    ]
    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
    return response

if __name__ == '__main__':
    # Enter the actual path of the original model.
    model_name = "path/to/model"
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.float16,
        device_map="auto"
    )
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    quant_res_root = 'you_output_path'
    # Quantization configuration
    dopt_config = f"/{quant_res_root}/dopt_config.json"
    # Quantization weight
    quanted_ckpt = f"/{quant_res_root}/train_output/trained.pth"
    model = get_quanted_model(
        model,
        dopt_config,
        quanted_ckpt
    )
    prompt = "who are you?"
    response = generate(prompt)
    print(response)