`# 安装核心依赖库

pip install torch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 --index-url https://download.pytorch.org/whl/cu121
pip install transformers==4.47.1
pip install trl
pip install peft
pip install datasets`

`from transformers import AutoTokenizer, AutoModelForCausalLM

import torch

# 加载轻量化模型和分词器

model_name = "gpt2" # 或 "Qwen2.5-Math-1.5B" 等
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)

# 设置 padding token

if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token`

`def extract_answer(text):
    """从模型输出中提取答案"""
    try:
        answer_part = text.split("<answer>")[1].split("</answer>")[0]
        return answer_part.strip()
    except IndexError:
        return ""

# 定义系统提示

SYSTEM_PROMPT = "Respond in the following format: <reasoning>...</reasoning><answer>...</answer>"`

`def correctness_reward(completions, human_answers, **kwargs):
    """正确性奖励函数"""
    rewards = []
    for completion, human_answer in zip(completions, human_answers):
        model_answer = extract_answer(completion)
        reward = 1.0 if model_answer == human_answer else -1.0
        rewards.append(reward)
    return rewards

def formatting*reward(completions, \*\*kwargs):
"""格式遵循奖励函数"""
return [0.5 for * in completions] # 简化版本`

`from peft import LoraConfig, get_peft_model, TaskType

from trl import GRPOConfig, GRPOTrainer

# 配置 LoRA

lora_config = LoraConfig(
r=8,
lora_alpha=16,
lora_dropout=0.05,
task_type="CAUSAL_LM",
)

# 应用 LoRA 到模型

model = get_peft_model(model, lora_config)

# 配置 GRPO 训练参数

training_args = GRPOConfig(
batch_size=4,
llm_batch_size=4,
num_epochs=5,
learning_rate=3e-4,
)`

`from datasets import load_dataset

# 加载 GSM8K 数据集

def get_gsm8k_questions(split="train"):
dataset = load_dataset("gsm8k", "main", split=split)

    def extract_xml_answer(example):
        answer_text = example["answer"].split("####")[-1].strip()
        return {
            "prompt": [
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": example["question"]}
            ],
            "answer": answer_text
        }

    return dataset.map(extract_xml_answer)

train_dataset = get_gsm8k_questions(split="train")

# 初始化 GRPOTrainer

trainer = GRPOTrainer(
model=model,
args=training_args,
train_dataset=train_dataset,
tokenizer=tokenizer,
peft_config=lora_config,
correctness_reward=correctness_reward,
formatting_reward=formatting_reward,
)

# 启动训练

trainer.train()

# 保存训练好的模型

trainer.save_model("rlvr_tuned_model")`