NLP中的迁移学习：原理与实践

雷帝木木

2人浏览 · 2026-03-25 21:57:11

雷帝木木 · 2026-03-25 21:57:11 发布

NLP中的迁移学习：原理与实践

背景

迁移学习在自然语言处理（NLP）领域取得了巨大成功，特别是自BERT等预训练语言模型出现以来。本文将深入探讨NLP中迁移学习的原理，介绍常用的预训练模型，并提供实践案例。

迁移学习原理

基本概念

迁移学习是一种机器学习方法，通过将从一个任务（源任务）中学到的知识应用到另一个相关任务（目标任务）中，以提高目标任务的性能。

NLP中的迁移学习范式

预训练-微调范式：
- 在大规模无标注语料上进行预训练
- 在特定任务的标注数据上进行微调
特征提取范式：
- 使用预训练模型作为特征提取器
- 将提取的特征用于下游任务

预训练语言模型

1. BERT及其变体

BERT（Bidirectional Encoder Representations from Transformers）是Google于2018年提出的预训练语言模型，采用双向Transformer编码器。

from transformers import BertTokenizer, BertForSequenceClassification
import torch

# 加载预训练模型和分词器
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

# 准备输入文本
text = "This is a sample sentence for classification."

# 分词
inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True)

# 前向传播
outputs = model(**inputs)
logits = outputs.logits

# 获取预测结果
predicted_class = torch.argmax(logits, dim=1).item()
print(f"Predicted class: {predicted_class}")

2. GPT系列模型

GPT（Generative Pre-trained Transformer）是OpenAI提出的生成式预训练语言模型，采用自回归的单向Transformer解码器。

from transformers import GPT2Tokenizer, GPT2LMHeadModel

# 加载预训练模型和分词器
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# 准备输入文本
prompt = "Once upon a time"

# 分词
inputs = tokenizer(prompt, return_tensors='pt')

# 生成文本
outputs = model.generate(
    inputs.input_ids,
    max_length=50,
    num_return_sequences=1,
    no_repeat_ngram_size=2,
    do_sample=True,
    temperature=0.7
)

# 解码生成的文本
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(f"Generated text: {generated_text}")

3. T5模型

T5（Text-to-Text Transfer Transformer）将所有NLP任务统一为文本到文本的格式，具有很强的通用性。

from transformers import T5Tokenizer, T5ForConditionalGeneration

# 加载预训练模型和分词器
tokenizer = T5Tokenizer.from_pretrained('t5-base')
model = T5ForConditionalGeneration.from_pretrained('t5-base')

# 准备输入文本（摘要任务）
input_text = "summarize: The quick brown fox jumps over the lazy dog."

# 分词
inputs = tokenizer(input_text, return_tensors='pt')

# 生成摘要
outputs = model.generate(inputs.input_ids, max_length=20)

# 解码生成的摘要
summary = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(f"Summary: {summary}")

迁移学习实践

1. 情感分析任务

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import BertTokenizer, BertForSequenceClassification, AdamW
from sklearn.model_selection import train_test_split
import pandas as pd

# 自定义数据集类
class SentimentDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_length):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_length = max_length
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, idx):
        text = self.texts[idx]
        label = self.labels[idx]
        
        encoding = self.tokenizer(
            text,
            add_special_tokens=True,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'label': torch.tensor(label, dtype=torch.long)
        }

# 加载数据
data = pd.read_csv('sentiment_data.csv')
texts = data['text'].tolist()
labels = data['label'].tolist()

# 分割数据
train_texts, val_texts, train_labels, val_labels = train_test_split(
    texts, labels, test_size=0.2, random_state=42
)

# 初始化分词器和模型
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

# 创建数据集和数据加载器
train_dataset = SentimentDataset(train_texts, train_labels, tokenizer, max_length=128)
val_dataset = SentimentDataset(val_texts, val_labels, tokenizer, max_length=128)

train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)

# 训练模型
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

optimizer = AdamW(model.parameters(), lr=2e-5)
criterion = torch.nn.CrossEntropyLoss()

for epoch in range(3):
    model.train()
    train_loss = 0
    
    for batch in train_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['label'].to(device)
        
        optimizer.zero_grad()
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        train_loss += loss.item()
        
        loss.backward()
        optimizer.step()
    
    # 验证模型
    model.eval()
    val_loss = 0
    correct = 0
    total = 0
    
    with torch.no_grad():
        for batch in val_loader:
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            labels = batch['label'].to(device)
            
            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
            loss = outputs.loss
            val_loss += loss.item()
            
            logits = outputs.logits
            predicted = torch.argmax(logits, dim=1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    
    accuracy = correct / total
    print(f"Epoch {epoch+1}, Train Loss: {train_loss/len(train_loader):.4f}, Val Loss: {val_loss/len(val_loader):.4f}, Val Accuracy: {accuracy:.4f}")

# 保存模型
model.save_pretrained('sentiment_model')
tokenizer.save_pretrained('sentiment_model')

2. 命名实体识别任务

import torch
from transformers import BertTokenizer, BertForTokenClassification
from torch.utils.data import Dataset, DataLoader

# 自定义数据集类
class NERDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_length):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_length = max_length
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, idx):
        text = self.texts[idx]
        label = self.labels[idx]
        
        # 分词并处理标签
        tokenized_input = self.tokenizer(
            text.split(),
            is_split_into_words=True,
            add_special_tokens=True,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        
        # 处理标签对齐
        word_ids = tokenized_input.word_ids()
        label_ids = []
        previous_word_idx = None
        
        for word_idx in word_ids:
            if word_idx is None:
                label_ids.append(-100)  # 特殊标记的标签
            elif word_idx != previous_word_idx:
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)  # 子词的标签
            previous_word_idx = word_idx
        
        return {
            'input_ids': tokenized_input['input_ids'].flatten(),
            'attention_mask': tokenized_input['attention_mask'].flatten(),
            'labels': torch.tensor(label_ids, dtype=torch.long)
        }

# 加载数据（示例数据）
texts = [
    ["John", "lives", "in", "New", "York"],
    ["Alice", "works", "at", "Google"]
]
labels = [
    [1, 0, 0, 2, 2],  # B-PER, O, O, B-LOC, I-LOC
    [1, 0, 0, 3]       # B-PER, O, O, B-ORG
]

# 初始化分词器和模型
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForTokenClassification.from_pretrained('bert-base-uncased', num_labels=4)

# 创建数据集和数据加载器
dataset = NERDataset(texts, labels, tokenizer, max_length=128)
dataloader = DataLoader(dataset, batch_size=1, shuffle=True)

# 训练模型
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5)

for epoch in range(3):
    model.train()
    for batch in dataloader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        
        optimizer.zero_grad()
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        loss.backward()
        optimizer.step()
        
        print(f"Epoch {epoch+1}, Loss: {loss.item():.4f}")

# 测试模型
model.eval()
test_text = ["Bob", "works", "at", "Microsoft", "in", "Seattle"]

with torch.no_grad():
    tokenized_input = tokenizer(
        test_text,
        is_split_into_words=True,
        add_special_tokens=True,
        return_tensors='pt'
    ).to(device)
    
    outputs = model(**tokenized_input)
    predictions = torch.argmax(outputs.logits, dim=2)
    
    # 解码预测结果
    word_ids = tokenized_input.word_ids()
    predicted_labels = []
    previous_word_idx = None
    
    for i, word_idx in enumerate(word_ids):
        if word_idx is not None and word_idx != previous_word_idx:
            predicted_labels.append(predictions[0, i].item())
        previous_word_idx = word_idx
    
    print(f"Test text: {test_text}")
    print(f"Predicted labels: {predicted_labels}")

模型性能对比

模型	情感分析准确率	命名实体识别F1值	文本分类准确率	机器翻译BLEU值
BERT-base	92.5%	91.2%	94.1%	-
RoBERTa-base	93.1%	92.0%	94.5%	-
GPT-2	91.8%	89.5%	93.2%	-
T5-base	92.2%	90.8%	93.8%	28.5
DistilBERT	91.5%	90.1%	92.8%	-

迁移学习优化策略

1. 模型选择

任务类型：根据任务类型选择合适的预训练模型
计算资源：根据可用资源选择模型大小
数据量：数据量较小时选择较小的模型，数据量较大时选择较大的模型

2. 微调策略

学习率调整：使用较小的学习率（如2e-5到5e-5）
批次大小：根据GPU内存调整批次大小
训练轮数：通常3-5轮即可
梯度裁剪：防止梯度爆炸

3. 数据增强

from transformers import pipeline
import random

# 使用回译进行数据增强
translator_en_fr = pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr")
translator_fr_en = pipeline("translation", model="Helsinki-NLP/opus-mt-fr-en")

def back_translate(text):
    # 英文 -> 法文
    fr_text = translator_en_fr(text)[0]['translation_text']
    # 法文 -> 英文
    en_text = translator_fr_en(fr_text)[0]['translation_text']
    return en_text

# 示例
original_text = "This is a sample sentence for data augmentation."
augmented_text = back_translate(original_text)
print(f"Original: {original_text}")
print(f"Augmented: {augmented_text}")