Editing Fine-tuning Large Language Models (section)

== <span style="color: #FFFFFF;">Applying</span> ==
'''LoRA fine-tuning with HuggingFace + PEFT:'''

<syntaxhighlight lang="python">
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model, TaskType
from trl import SFTTrainer
import datasets

# Load base model (quantized for efficiency)
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    load_in_4bit=True,      # QLoRA: quantize to 4-bit
    device_map="auto"
)

# LoRA configuration
lora_config = LoraConfig(
    r=16,                           # Rank
    lora_alpha=32,                  # Scaling factor
    target_modules=["q_proj", "v_proj"],  # Which layers to adapt
    lora_dropout=0.05,
    bias="none",
    task_type=TaskType.CAUSAL_LM
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: 4,194,304 || all params: 6,742,609,920 || trainable%: 0.06%

# Training setup
training_args = TrainingArguments(
    output_dir="./finetuned_model",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    fp16=True,
    save_steps=100,
    logging_steps=25,
)

# Dataset: each sample has "text" field with full formatted prompt+response
dataset = datasets.load_dataset("json", data_files="train.jsonl")["train"]

trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
)
trainer.train()
</syntaxhighlight>

; Data format for instruction tuning (Llama chat template)
: '''System''' → Defines the model's role and constraints
: '''User turn''' → The instruction or question
: '''Assistant turn''' → The desired response (what the model learns to produce)
: '''Special tokens''' → [INST], [/INST], <<SYS>> etc. must exactly match the model's chat template
</div>

<div style="background-color: #8B4500; color: #FFFFFF; padding: 20px; border-radius: 8px; margin-bottom: 15px;">