使用GpuGeek高效完成LLaMA大模型微调：实践与心得分享

from transformers import LlamaTokenizer, LlamaForCausalLM, TrainingArguments, Trainer
from peft import get_peft_model, LoraConfig, TaskType
from datasets import load_dataset

# 加载模型与Tokenizer（模型市场一键加载路径）
model = LlamaForCausalLM.from_pretrained("/mnt/models/llama-7b-hf")
tokenizer = LlamaTokenizer.from_pretrained("/mnt/models/llama-7b-hf")

# 加载数据集
dataset = load_dataset("json", data_files="/mnt/datasets/alpaca-cleaned.json")['train']

# 预处理
def tokenize(example):
    prompt = f"### 指令: {example['instruction']}\n### 输入: {example['input']}\n### 回答: {example['output']}"
    return tokenizer(prompt, truncation=True, padding="max_length", max_length=512)

tokenized_dataset = dataset.map(tokenize)

# 配置LoRA
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    inference_mode=False,
    r=8,
    lora_alpha=32,
    lora_dropout=0.1
)

model = get_peft_model(model, lora_config)

# 训练配置
training_args = TrainingArguments(
    output_dir="./output",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=2,
    num_train_epochs=2,
    learning_rate=2e-4,
    fp16=True,
    logging_steps=20,
    save_steps=200,
    save_total_limit=2,
    report_to="none"
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset
)

trainer.train()

python generate.py \
  --model_path ./output/checkpoint-final \
  --prompt "请用中文简述Transformer原理" \
  --max_new_tokens 128