Keras文本数据预处理范例——IMDB影评情感分类

from keras.preprocessing.text import Tokenizer
from tqdm import tqdm 

# 数据集路径
train_data_path = 'imdb_datasets/xx_train_imdb'
test_data_path = 'imdb_datasets/xx_test_imdb'

train_samples =  #训练集样本数量
test_samples =  #测试集样本数量

max_words =   # 保留词频最高的前10000个词
maxlen =        # 每个样本文本内容最多保留500个词

# 构建训练集文本生成器
def texts_gen():
    with open(train_data_path,'r',encoding = 'utf-8') as f,
    tqdm(total = train_samples) as pbar:      
        while True:
            text = (f.readline().rstrip('n').split('t')[-1])
            if not text:
                break
            if len(text) > maxlen:
                text = text[:maxlen]
            pbar.update()
            yield text

texts = texts_gen()
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(texts)

import os

scatter_train_data_path = 'imdb_datasets/train/'
scatter_test_data_path = 'imdb_datasets/test/'

# 将数据按样本打散到多个文件
def scatter_data(data_file, scatter_data_path):
    if not os.path.exists(scatter_data_path):
        os.makedirs(scatter_data_path)
    for idx,line in tqdm(enumerate(open(data_file,'r',encoding = 'utf-8'))):
        with open(scatter_data_path + str(idx) + '.txt','w',
        encoding = 'utf-8') as f:
             f.write(line)

scatter_data(train_data_path,scatter_train_data_path)
scatter_data(test_data_path,scatter_test_data_path)

# 定义Sequence数据管道， 可以多线程读数据

import keras
import numpy as np
from keras.preprocessing.sequence import pad_sequences

batch_size = 

class DataGenerator(keras.utils.Sequence):

    def __init__(self,n_samples,data_path,batch_size=batch_size,shuffle=True):
        self.data_path = data_path
        self.n_samples = n_samples
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.on_epoch_end()

    def __len__(self):
        return int(np.ceil(self.n_samples/self.batch_size))

    def __getitem__(self, index):
        # Generate indexes of the batch
        batch_indexes = self.indexes[index*self.batch_size:(index+)*self.batch_size]
        # Generate data
        datas, labels = self.__data_generation(batch_indexes)
        return datas, labels

    def on_epoch_end(self):
        self.indexes = np.arange(self.n_samples)
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __read_file(self,file_name):
        with open(file_name,encoding = 'utf-8') as f:
            line = f.readline()
        return line

    def __data_generation(self, batch_indexes):
        lines = [self.__read_file(self.data_path + str(i) + '.txt') for i in batch_indexes]
        labels = np.array([int(line.strip().split('t')[]) for line in lines])
        texts = [line.strip().split('t')[-1] for line in lines]
        sequences = tokenizer.texts_to_sequences(texts)
        datas = pad_sequences(sequences,maxlen)

        return datas,labels


train_gen = DataGenerator(train_samples,scatter_train_data_path)
test_gen = DataGenerator(test_samples,scatter_test_data_path)

from keras import models,layers
from keras import backend as K
K.clear_session()

embedding_dim = 

model = models.Sequential()
model.add(layers.Embedding(max_words, embedding_dim, input_length=maxlen))
model.add(layers.Flatten())
model.add(layers.Dense(,activation = 'relu'))
model.add(layers.Dense(, activation = 'sigmoid'))
model.summary()

model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['acc'])

epoch_num = 
steps_per_epoch = int(np.ceil(train_samples/batch_size))
validation_steps = int(np.ceil(test_samples/batch_size))

history = model.fit_generator(train_gen,
                         steps_per_epoch = steps_per_epoch,
                         epochs = epoch_num,
                         validation_data= test_gen,
                         validation_steps = validation_steps,
                         workers=,
                         use_multiprocessing=False #linux上可使用多进程读取数据
                         )

import os
import pandas as pd

# 保存得分
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(, len(acc) + )
dfhistory  = pd.DataFrame({'epoch':epochs,'train_loss':loss,'test_loss':val_loss,
                  'train_acc':acc,'test_acc':val_acc})

print(dfhistory)

model.save('imdb_model.h5')