NLP | 百度 ERNIE - 从0开始快速上手

!git clone -b dygraph --single-branch https://github.com/PaddlePaddle/ERNIE.git

!pip install -r ERNIE/requirements.txt

!wget https://ernie-github.cdn.bcebos.com/data-chnsenticorp.tar.gz
!tar xvf data-chnsenticorp.tar.gz

seg_a label
选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，但还算整洁。泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。包的早餐是西式的，还算丰富。服务吗，一般       1
15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错    1
房间太小。其他的都一般。。。。。。。。。0
1.接电源没有几分钟,电源适配器热的不行. 2.摄像头用不起来. 3.机盖的钢琴漆，手不能摸，一摸一个印. 4.硬盘分区不好办.        0
今天才知道这书还有第6卷,真有点郁闷:为什么同一套书有两种版本呢?当当网是不是该跟出版社商量商量,单独出个第6卷,让我们的孩子不会有所遗憾。1
机器背面似乎被撕了张什么标签，残胶还在。但是又看不出是什么标签不见了，该有的都在，怪    0
呵呵，虽然表皮看上去不错很精致，但是我还是能看得出来是盗的。但是里面的内容真的不错，我妈爱看，我自己也学着找一些穴位。0
这本书实在是太烂了,以前听浙大的老师说这本书怎么怎么不对,哪些地方都是误导的还不相信,终于买了一本看一下,发现真是~~~无语,这种书都写得出来  0
地理位置佳，在市中心。酒店服务好、早餐品种丰富。我住的商务数码房电脑宽带速度满意,房间还算干净，离湖南路小吃街近。1

!export CUDA_VISIBLE_DEVICES=0 
!PYTHONPATH=./ERNIE python ./ERNIE/ernie/finetune_sementic_analysis_dygraph.py \
        --from_pretrained ernie-1.0 \
        --data_dir ./chnsenticorp/ \
        --epoch 10 \
        --lr 5e-5 \
        --bsz 32 \
        --max_steps $((9600*10/32)) \
        --save_dir ./tuned_model

training: 250it [01:39,  2.96it/s]2020-05-15 17:52:21,377-DEBUG: train loss 0.00880 lr 3.585e-05
training: 260it [01:43,  3.00it/s]2020-05-15 17:52:24,743-DEBUG: train loss 0.05025 lr 3.568e-05
training: 270it [01:46,  3.00it/s]2020-05-15 17:52:28,108-DEBUG: train loss 0.06813 lr 3.552e-05
training: 280it [01:49,  3.00it/s]2020-05-15 17:52:31,474-DEBUG: train loss 0.12881 lr 3.535e-05
training: 290it [01:53,  3.00it/s]2020-05-15 17:52:34,840-DEBUG: train loss 0.06156 lr 3.518e-05
2020-05-15 17:52:42,877-DEBUG: acc 0.93250
training: 10it [00:08,  1.88it/s]2020-05-15 17:52:46,317-DEBUG: train loss 0.00679 lr 3.485e-05
training: 20it [00:11,  2.84it/s]2020-05-15 17:52:49,817-DEBUG: train loss 0.13993 lr 3.468e-05
training: 30it [00:15,  2.89it/s]2020-05-15 17:52:53,297-DEBUG: train loss 0.02414 lr 3.452e-05

!head ./chnsenticorp/dev/part.0|awk -F"\t" '{print $1}'| PYTHONPATH=./ERNIE  python ./ERNIE/ernie/finetune_sementic_analysis_dygraph.py \
        --from_pretrained ernie-1.0 \
        --data_dir ./chnsenticorp/ \
        --epoch 10 \
        --lr 5e-5 \
        --bsz 32 \
        --eval \
        --max_steps $((9600*10/32)) \
        --save_dir ./tuned_model

import sys
sys.path.append('./ERNIE')
import numpy as np
from sklearn.metrics import f1_score
import paddle as P
import paddle.fluid as F
import paddle.fluid.layers as L
import paddle.fluid.dygraph as D

from ernie.tokenizing_ernie import ErnieTokenizer
from ernie.modeling_ernie import ErnieModelForSequenceClassification

BATCH=32
MAX_SEQLEN=300
LR=5e-5
EPOCH=10

D.guard().__enter__() # 为了让Paddle进入动态图模式，需要添加这一行在最前面

ernie = ErnieModelForSequenceClassification.from_pretrained('ernie-1.0', num_labels=3)
optimizer = F.optimizer.Adam(LR, parameter_list=ernie.parameters())
tokenizer = ErnieTokenizer.from_pretrained('ernie-1.0')

def make_data(path):
    data = []
    for i, l in enumerate(open(path)):
        if i == 0:
            continue
        l = l.strip().split('\t')
        text, label = l[0], int(l[1])
        text_id, _ = tokenizer.encode(text) # ErnieTokenizer 会自动添加ERNIE所需要的特殊token，如[CLS], [SEP]
        text_id = text_id[:MAX_SEQLEN]
        text_id = np.pad(text_id, [0, MAX_SEQLEN-len(text_id)], mode='constant') # 对所有句子都补长至300，这样会比较费显存；
        label_id = np.array(label+1)
        data.append((text_id, label_id))
    return data

train_data = make_data('./chnsenticorp/train/part.0')
test_data = make_data('./chnsenticorp/dev/part.0')

def get_batch_data(data, i):
    d = data[i*BATCH: (i + 1) * BATCH]
    feature, label = zip(*d)
    feature = np.stack(feature)  # 将BATCH行样本整合在一个numpy.array中
    label = np.stack(list(label))
    feature = D.to_variable(feature) # 使用to_variable将numpy.array转换为paddle tensor
    label = D.to_variable(label)
    return feature, label

for i in range(EPOCH):
    np.random.shuffle(train_data) # 每个epoch都shuffle数据以获得最佳训练效果；
    #train
    for j in range(len(train_data) // BATCH):
        feature, label = get_batch_data(train_data, j)
        loss, _ = ernie(feature, labels=label) # ernie模型的返回值包含(loss, logits)；其中logits目前暂时不需要使用
        loss.backward()
        optimizer.minimize(loss)
        ernie.clear_gradients()
        if j % 10 == 0:
            print('train %d: loss %.5f' % (j, loss.numpy()))
        # evaluate
        if j % 100 == 0:
            all_pred, all_label = [], []
            with D.base._switch_tracer_mode_guard_(is_train=False): # 在这个with域内ernie不会进行梯度计算；
                ernie.eval() # 控制模型进入eval模式，这将会关闭所有的dropout；
                for j in range(len(test_data) // BATCH):
                    feature, label = get_batch_data(test_data, j)
                    loss, logits = ernie(feature, labels=label) 
                    all_pred.extend(L.argmax(logits, -1).numpy())
                    all_label.extend(label.numpy())
                ernie.train()
            f1 = f1_score(all_label, all_pred, average='macro')
            acc = (np.array(all_label) == np.array(all_pred)).astype(np.float32).mean()
            print('acc %.5f' % acc)

train 0: loss 0.05833
acc 0.91723
train 10: loss 0.03602
train 20: loss 0.00047
train 30: loss 0.02403
train 40: loss 0.01642
train 50: loss 0.12958
train 60: loss 0.04629
train 70: loss 0.00942
train 80: loss 0.00068
train 90: loss 0.05485
train 100: loss 0.01527
acc 0.92821
train 110: loss 0.00927
train 120: loss 0.07236
train 130: loss 0.01391
train 140: loss 0.01612