第22天：NLP实战(六)——基于PaddleHub的疫情期间网民情绪识别

!cd data/data22724 && unzip -o test_dataset.zip
!cd data/data22724 && unzip -o "train_ dataset.zip"

# 转换编码
def re_encode(path):
    with open(path, 'r', encoding='GB2312', errors='ignore') as file:
        lines = file.readlines()
    with open(path, 'w', encoding='utf-8') as file:
        file.write(''.join(lines))
        
re_encode('data/data22724/nCov_10k_test.csv') # 这里写你自己数据集的路径
re_encode('data/data22724/nCoV_100k_train.labled.csv') # 写自己文件的路径

# 读取数据
import pandas as pd
train_labled = pd.read_csv('data/data22724/nCoV_100k_train.labled.csv', engine ='python')
test = pd.read_csv('data/data22724/nCov_10k_test.csv', engine ='python')
print(train_labled.shape)
print(test.shape)
print(train_labled.columns)

train_labled.head(3)
test.head(3)

%matplotlib inline
train_labled['情感倾向'].value_counts(normalize=True).plot(kind='bar');

# 清除异常标签数据
train_labled = train_labled[train_labled['情感倾向'].isin(['-1','0','1'])]

train_labled['微博中文内容'].str.len().describe()

!pip install  paddlehub==1.6.0 -i https://pypi.tuna.tsinghua.edu.cn/simple

# 划分验证集，保存格式  text[t]label
from sklearn.model_selection import train_test_split
train_labled = train_labled[['微博中文内容', '情感倾向']]
train, valid = train_test_split(train_labled, test_size=0.2, random_state=2020)
train.to_csv('/home/aistudio/data/data22724/train.txt', index=False, header=False, sep='t')
valid.to_csv('/home/aistudio/data/data22724/valid.txt', index=False, header=False, sep='t')

# 自定义数据集
import os
import codecs
import csv

from paddlehub.dataset.base_nlp_dataset import BaseNLPDataset

class MyDataset(BaseNLPDataset):
    """DemoDataset"""
    def __init__(self):
        # 数据集存放位置
        self.dataset_dir = "/home/aistudio/data/data22724"
        super(MyDataset, self).__init__(
            base_path=self.dataset_dir,
            train_file="train.txt",
            dev_file="valid.txt",
            test_file="valid.txt",
            train_file_with_header=False,
            dev_file_with_header=False,
            test_file_with_header=False,
            # 数据集类别集合
            label_list=["-1", "0", "1"])

dataset = MyDataset()
for e in dataset.get_train_examples()[:3]:
    print("{}t{}t{}".format(e.guid, e.text_a, e.label))

# 加载模型
import paddlehub as hub
module = hub.Module(name="ernie")

# 构建Reader
reader = hub.reader.ClassifyReader(
    dataset=dataset,
    vocab_path=module.get_vocab_path(),
    sp_model_path = module.get_spm_path(),
    word_dict_path = module.get_word_dict_path(),
    max_seq_len=128
)

# finetune策略1
strategy = hub.L2SPFinetuneStrategy(
    learning_rate=1e-4,
    optimizer_name="adam",
    regularization_coeff=1e-3)

# 运行配置
config = hub.RunConfig(
    use_cuda=True,
    num_epoch= 10,
    batch_size= 128,
    checkpoint_dir="hub_finetune",
    strategy=strategy
    )

# Finetune Task
inputs, outputs, program = module.context(
    trainable=True, max_seq_len= 128)

# Use "pooled_output" for classification tasks on an entire sentence.
pooled_output = outputs["pooled_output"]

feed_list = [
    inputs["input_ids"].name,
    inputs["position_ids"].name,
    inputs["segment_ids"].name,
    inputs["input_mask"].name,
]

cls_task = hub.TextClassifierTask(
        data_reader=reader,
        feature=pooled_output,
        feed_list=feed_list,
        num_classes=dataset.num_labels,
        config=config,
        metrics_choices=["f1"])

# finetune
run_states = cls_task.finetune_and_eval()

# 预测
import numpy as np
    
inv_label_map = {val: key for key, val in reader.label_map.items()}

# Data to be prdicted
data = test[['微博中文内容']].fillna(' ').values.tolist()

run_states = cls_task.predict(data=data)
results = [run_state.run_results for run_state in run_states]

# 生成预测结果
proba = np.vstack([r[0] for r in results])
prediction = list(np.argmax(proba, axis=1))
prediction = [inv_label_map[p] for p in prediction]
        
submission = pd.DataFrame()
submission['id'] = test['微博id'].values
submission['id'] = submission['id'].astype(str) + ' '
submission['y'] = prediction
np.save('proba.npy', proba)
submission.to_csv('result_ernie.csv', index=False)
submission.head()

submission['text'] = test[['微博中文内容']].fillna(' ').values
submission['label'] = submission['y'].values
display(submission[['text', 'label']][175:180])