# 加载大模型
from vllm import LLM
model = LLM(model=args.model,
            trust_remote_code=True,
            quantization=args.quantization,     # "ifq代表int8 weight-only量化，ifq_int4代表int4 weight-only量化，smoothquant代表int8 smoothquant量化，默认None代表不使用量化加速"
            kv_cache_dtype=args.kv_cache_dtype, # "int8代表使用int8精度压缩kv-cache，默认auto代表不使用kv-cache量化"
            dtype=torch.float16)

# 定义采样参数
sampling_params = SamplingParams(top_k=50, top_p=0.8, max_tokens=args.output_len)

# 推理
outputs = model.generate(prompt_token_ids=input_ids, sampling_params=sampling_params)

sampling_params = SamplingParams(seed = 10, top_k=50, use_beam_search = False)
llm = LLM(model = model_path,
          enforce_eager = True,
          quantization = None)
output = llm.generate(prompts, sampling_params = sampling_params)

llm = LLM(model = model_path,
          enforce_eager = True,
          use_lookahead = True,        #开启lookahead并行解码
          use_v2_block_manager = True, #lookahead并行解码需要使用block_manager v2
          num_speculative_tokens = 12, #一次并行解码长度，单BS可以设置成12， 多BS可以适当降低一次解码长度，比如设置成8或6
          quantization = None)         #并行解码也支持int8量化，以及smoothquant量化，设置quantization='ifq' or 'smoothquant'打开相应
量化。模型量化使用方法参考模型量化部分。

    llm = LLM(model = model_path,
          enforce_eager = True,
          use_v2_block_manager = True,
          num_speculative_tokens = 12,
          ngram_prompt_lookup_max = 12,
          ngram_prompt_lookup_min = 2,
          speculative_model = "[ngram]",
          quantization = None)