STI比赛任务一：【智能问答baseline】

问题query：渝北区面积

篇章doc_text：重庆主城九区面积 【导语】:重庆市主城九区面积为:4779平方千米。其中渝中区面积为23....	

答案answer：[渝北区幅员面积1452平方公里, 2、渝北区面积: 全区幅员面积1452平方公里。] [53, 225]

问题q：备孕偶尔喝冰的可以吗

篇章d：备孕能吃冷的食物吗 炎热的夏天让很多人都觉得闷热...,下面一起来看看吧! 备孕能吃冷的食物吗 在中医养生中,女性体质属阴,不可以贪凉。吃了过多寒凉、生冷的食物后,会消耗阳气,导致寒邪内生,侵害子宫。另外,宫寒是肾阳虚的表现,不会直接导致不孕。但宫寒会引起妇科疾病,所以也不可不防。因此处于备孕期的女性最好不要吃冷的食物。 备孕食谱有哪些 ...

答案a：在中医养生中,女性体质属阴,不可以贪凉。吃了过多寒凉、生冷的食物后,会消耗阳气,导致寒邪内生,侵害子宫。另外,宫寒是肾阳虚的表现,不会直接导致不孕。但宫寒会引起妇科疾病,所以也不可不防。因此处于备孕期的女性最好不要吃冷的食物。

!python -m pip install paddlepaddle-gpu==2.3.0.post101 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html
!pip install --upgrade paddlenlp

import pandas as pd
import paddle
from paddlenlp.data import Stack, Dict, Pad
import paddlenlp
from paddlenlp.datasets import load_dataset
from functools import partial
import collections
import time
import json
import paddle
from paddlenlp.metrics.squad import squad_evaluate, compute_prediction
import pyarrow as pa
import pyarrow.dataset as ds
import pandas as pd
from datasets import Datase

!ls /home/aistudio/data/data174963

# 将比赛数据解压到work目录下
!if [ -d "work/data_task1/" ];then rm -f -r work/data_task1/;fi
!tar -zxvf /home/aistudio/data/data174963/data_task1.tar.gz -C /home/aistudio/work/
!ls work/data_task1/
!tree work/data_task1/

train_examples = Dataset(pa.Table.from_pandas(train))
dev_examples = Dataset(pa.Table.from_pandas(dev))
train_examples,dev_examples

由于文章加问题的文本长度可能大于max_seq_length，答案出现的位置有可能出现在文章最后，所以不能简单的对文章进行截断。

那么对于过长的文章，则采用滑动窗口将文章分成多段，分别与问题组合。再用对应的tokenizer转化为模型可接受的feature。doc_stride参数就是每次滑动的距离。滑动窗口生成InputFeature的过程如下图：

from functools import partial

max_seq_length = 512
doc_stride = 128

train_trans_func = partial(prepare_train_features, 
                           max_seq_length=max_seq_length, 
                           doc_stride=doc_stride,
                           tokenizer=tokenizer)


dev_trans_func = partial(prepare_validation_features, 
                           max_seq_length=max_seq_length, 
                           doc_stride=doc_stride,
                           tokenizer=tokenizer)
    
column_names = train_examples.column_names

train_ds = train_examples.map(train_trans_func, batched=True, num_proc=3, remove_columns=column_names)
dev_ds = dev_examples.map(dev_trans_func, batched=True, num_proc=3, remove_columns=column_names)

import paddle
from paddlenlp.data import DataCollatorWithPadding

batch_size = 64

# 定义BatchSampler
train_batch_sampler = paddle.io.DistributedBatchSampler(
        train_ds, batch_size=batch_size, shuffle=True)

dev_batch_sampler = paddle.io.BatchSampler(
    dev_ds, batch_size=batch_size, shuffle=False)


# 定义batchify_fn

train_batchify_fn = DataCollatorWithPadding(tokenizer)

dev_batchify_fn = DataCollatorWithPadding(tokenizer)

# 构造DataLoader
train_data_loader = paddle.io.DataLoader(
    dataset=train_ds,
    batch_sampler=train_batch_sampler,
    collate_fn=train_batchify_fn,
    return_list=True)

dev_data_loader = paddle.io.DataLoader(
    dataset=dev_ds_for_model,
    batch_sampler=dev_batch_sampler,
    collate_fn=dev_batchify_fn,
    return_list=True)

from paddlenlp.transformers import AutoModelForQuestionAnswering

model = AutoModelForQuestionAnswering.from_pretrained(MODEL_NAME)

class CrossEntropyLossForRobust(paddle.nn.Layer):
    def __init__(self):
        super(CrossEntropyLossForRobust, self).__init__()

    def forward(self, y, label):
        start_logits, end_logits = y
        start_position, end_position = label
        start_position = paddle.unsqueeze(start_position, axis=-1)
        end_position = paddle.unsqueeze(end_position, axis=-1)
        start_loss = paddle.nn.functional.cross_entropy(
            input=start_logits, label=start_position)
        end_loss = paddle.nn.functional.cross_entropy(
            input=end_logits, label=end_position)
        loss = (start_loss + end_loss) / 2
        return loss

# 训练过程中的最大学习率
learning_rate = 3e-5 

# 训练轮次
epochs = 1

# 学习率预热比例
warmup_proportion = 0.1

# 权重衰减系数，类似模型正则项策略，避免模型过拟合
weight_decay = 0.01

num_training_steps = len(train_data_loader) * epochs

# 学习率衰减策略
lr_scheduler = paddlenlp.transformers.LinearDecayWithWarmup(learning_rate, num_training_steps, warmup_proportion)

decay_params = [
    p.name for n, p in model.named_parameters()
    if not any(nd in n for nd in ["bias", "norm"])
]
optimizer = paddle.optimizer.AdamW(
    learning_rate=lr_scheduler,
    parameters=model.parameters(),
    weight_decay=weight_decay,
    apply_decay_param_fun=lambda x: x in decay_params)

criterion = CrossEntropyLossForRobust()
global_step = 0
for epoch in range(1, epochs + 1):
    for step, batch in enumerate(train_data_loader, start=1):
        global_step += 1
        input_ids, segment_ids, start_positions, end_positions = batch
        logits = model(input_ids=batch["input_ids"], token_type_ids=batch["token_type_ids"])
        loss = criterion(logits, (batch["start_positions"], batch["end_positions"]))

        if global_step % 400 == 0 :
            print("global step %d, epoch: %d, batch: %d, loss: %.5f" % (global_step, epoch, step, loss))

        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.clear_grad()

test=pd.read_json('work/data_task1/test_data/test.json',lines=True)
test['id']=[f'test_{id}' for id in range(len(test))]
test['answer_list']=[[] for i in range(len(test))]
test['answer_start_list']=[[] for i in range(len(test))]
test_examples = Dataset(pa.Table.from_pandas(test))

test_trans_func = partial(prepare_validation_features, 
                           max_seq_length=max_seq_length, 
                           doc_stride=doc_stride,
                           tokenizer=tokenizer)
test_ds = test_examples.map(test_trans_func, batched=True, num_proc=3, remove_columns=['answer_list',
 'title',
 'url',
 'answer_start_list',
 'doc_text',
 'query',
 'id'])
test_ds_for_model = test_ds.remove_columns(["example_id", "offset_mapping"])


batch_size = 16


test_batch_sampler = paddle.io.BatchSampler(
    test_ds, batch_size=batch_size, shuffle=False)


# 定义batchify_fn

test_batchify_fn = DataCollatorWithPadding(tokenizer)


test_data_loader = paddle.io.DataLoader(
    dataset=test_ds_for_model,
    batch_sampler=test_batch_sampler,
    collate_fn=test_batchify_fn,
    return_list=True)
# 传入test_data_loader，并将is_test参数设为True，即可生成千言比赛可提交的结果。
all_predictions, all_nbest_json, scores_diff_json=evaluate(model=model, raw_dataset=test_examples, dataset=test_ds, data_loader=test_data_loader, is_test=True

sub_data=[]
for i in range(len(test_examples["query"])):
    prob=all_nbest_json[test_examples['id'][i]][0]['probability']
    text=all_nbest_json[test_examples['id'][i]][0]['text']
    sub_data.append([prob,text])
sub=pd.DataFrame(sub_data,columns=['prob','text'])
sub['text']=sub['text'].apply(lambda  x:"NoAnswer" if len(x.strip())<2 else x)
sub[['prob','text']].to_csv('subtask1_test_pred.txt',sep='\t',header=None,index=None)