Huggingface🤗NLP笔记6：数据集预处理，使用dynamic padding构造batch

# 先看看cuda是否可用
import torch
torch.cuda.is_available()

>>> True

from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification

# Same as before
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)

>>> 
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

sequences = [
    "I've been waiting for a HuggingFace course my whole life.",
    "This course is amazing!",
]
batch = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")
batch['labels'] = torch.tensor([1, 1])  # tokenizer出来的结果是一个dictionary，所以可以直接加入新的 key-value

optimizer = AdamW(model.parameters())
loss = model(**batch).loss  #这里的 loss 是直接根据 batch 中提供的 labels 来计算的，回忆：前面章节查看 model 的输出的时候，有loss这一项
loss.backward()
optimizer.step()

from datasets import load_dataset

raw_datasets = load_dataset("glue", "mrpc")
raw_datasets

DatasetDict({
    train: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 408
    })
    test: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 1725
    })
})

raw_train_dataset = raw_datasets['train']
raw_train_dataset[0]

{'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
 'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .',
 'label': 1,
 'idx': 0}

type(raw_train_dataset['sentence1'])  # 直接取出所有的sentence1，形成一个list

>>> list

raw_train_dataset.features

>>>
{'sentence1': Value(dtype='string', id=None),
 'sentence2': Value(dtype='string', id=None),
 'label': ClassLabel(num_classes=2, names=['not_equivalent', 'equivalent'], names_file=None, id=None),
 'idx': Value(dtype='int32', id=None)}

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')

tokenized_sentences_1 = tokenizer(raw_train_dataset['sentence1'])
tokenized_sentences_2 = tokenizer(raw_train_dataset['sentence2'])

from pprint import pprint as print
inputs = tokenizer("first sentence", "second one")
print(inputs)

>>>
{'attention_mask': [1, 1, 1, 1, 1, 1, 1],
 'input_ids': [101, 2034, 6251, 102, 2117, 2028, 102],
 'token_type_ids': [0, 0, 0, 0, 1, 1, 1]}

tokenizer.decode(inputs.input_ids)

>>>
'[CLS] first sentence [SEP] second one [SEP]'

tokenized_dataset = tokenizer(
    raw_datasets["train"]["sentence1"],
    raw_datasets["train"]["sentence2"],
    padding=True,
    truncation=True,
)

def tokenize_function(sample):
    # 这里可以添加多种操作，不光是tokenize
    # 这个函数处理的对象，就是Dataset这种数据类型，通过features中的字段来选择要处理的数据
    return tokenizer(sample['sentence1'], sample['sentence2'], truncation=True)

tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)
tokenized_datasets

DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 408
    })
    test: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 1725
    })
})

raw_datasets.map(
    function,
    with_indices: bool = False,
    input_columns: Union[str, List[str], NoneType] = None,
    batched: bool = False,
    batch_size: Union[int, NoneType] = 1000,
    remove_columns: Union[str, List[str], NoneType] = None,
    keep_in_memory: bool = False,
    load_from_cache_file: bool = True,
    cache_file_names: Union[Dict[str, Union[str, NoneType]], NoneType] = None,
    writer_batch_size: Union[int, NoneType] = 1000,
    features: Union[datasets.features.Features, NoneType] = None,
    disable_nullable: bool = False,
    fn_kwargs: Union[dict, NoneType] = None,
    num_proc: Union[int, NoneType] = None,  # 使用此参数，可以使用多进程处理
    desc: Union[str, NoneType] = None,
) -> 'DatasetDict'
Docstring:
Apply a function to all the elements in the table (individually or in batches)
and update the table (if function does updated examples).
The transformation is applied to all the datasets of the dataset dictionary.

features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids']

from transformers import DataCollatorWithPadding
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

samples = tokenized_datasets['train'][:5]
samples.keys()
# >>> ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids']
samples = {k:v for k,v in samples.items() if k not in ["idx", "sentence1", "sentence2"]}  # 把这里多余的几列去掉
samples.keys()
# >>> ['attention_mask', 'input_ids', 'label', 'token_type_ids']

# 打印出每个句子的长度：
[len(x) for x in samples["input_ids"]]

>>>
[50, 59, 47, 67, 59]

batch = data_collator(samples)  # samples中必须包含 input_ids 字段，因为这就是collator要处理的对象
batch.keys()
# >>> dict_keys(['attention_mask', 'input_ids', 'token_type_ids', 'labels'])

# 再打印长度：
[len(x) for x in batch['input_ids']]

>>>
[67, 67, 67, 67, 67]

{k:v.shape for k,v in batch.items()}

>>>
{'attention_mask': torch.Size([5, 67]),
 'input_ids': torch.Size([5, 67]),
 'token_type_ids': torch.Size([5, 67]),
 'labels': torch.Size([5])}