第13天：NLP补充——RNN算法

python3 -m pip install --upgrade torch torchvision -i https://pypi.tuna.tsinghua.edu.cn/simple

import torch
import torch.nn
class RnnCore(nn.Module):
	def __init__(self,inputSize,hiddenSize，outputSize，layers=1):
		super(RnnCore,self).__init__()
		self.hidden_size = hiddenSize
		self.num_layers = layers
		self.embedding = nn.Embedding(inputSize,hiddenSize)
		self.rnn = nn.RNN(hiddenSize,hiddenSize,layers,batch_first=True)
		self.fc = nn.Linear(hiddenSize,outputSize)
		self.softmax = nn.LogSoftmax(dim=1)
	def forward(self,inputs,hidden):
		x = self.embedding(inputs)
		output,hidden = self.rnn(x,hidden)
		output = output[:,-1,:]
		output = self.fc(output)
		return output,hidden
	def initHidden(self):
		return Variable(torch.zeros(self.num_layers,1,self,hidden_size))

#encoding=utf-8
import torch
import torch.nn  as nn
import torch.optim
from torch.autograd import Variable
from collections import Counter
import matplotlib 
import matplotlib.pyplot as plt
from matplotlib import rc
import numpy as np
class SimpleRnn(nn.Module):
    def __init__(self,input_size,hidden_size,output_size,num_layers=1):
        super(SimpleRnn,self).__init__()
        self.hidden_size = hidden_size
        self.num_layers  = num_layers
        self.embedding = nn.Embedding(input_size,hidden_size)
        self.rnn = nn.RNN(hidden_size,hidden_size,num_layers,batch_first= True)
        self.fc = nn.Linear(hidden_size,output_size)
        self.softmax = nn.LogSoftmax(dim=1)
    def forward(self,inputs,hidden):
        x = self.embedding(inputs)
        output,hidden = self.rnn(x,hidden)
        output = output[:,-1,:]
        output = self.fc(output)
        output = self.softmax(output)  
        return output,hidden
    def initHidden(self):
        return Variable(torch.zeros(self.num_layers,1,self.hidden_size))
train_set = []
validset = []
sample = 2000
sz = 10
probablity = 1.0 *np.array([10,6,4,3,1,1,1,1,1,1])
probablity = probablity[:sz]
probablity = probablity / sum(probablity)
for m in range(2000):
    n = np.random.choice(range(1,sz+1),p=probablity)
    inputs = [0]*n + [1]*n
    inputs.insert(0,3)
    inputs.append(2)
    train_set.append(inputs)
for m in range(sample // 10):
    n =np.random.choice(range(1,sz+1),p=probablity)
    inputs = [0] * n + [1] *n
    inputs.insert(0,3)
    inputs.append(2)
    validset.append(inputs)
for m in range(2):
    n = sz + m
    inputs = [0] * n + [1] *n
    inputs.insert(0,3)
    inputs.append(2)
    validset.append(inputs)
rnn = SimpleRnn(input_size=4, hidden_size=2, output_size=3)
criterion = torch.nn.NLLLoss() 
optimizer = torch.optim.Adam(rnn.parameters(),lr=0.001)
num_epoch = 50
results = []
for epoch in range(num_epoch):
    train_loss = 0
    np.random.shuffle(train_set)
    for i,seq in enumerate(train_set):
        loss = 0
        hidden = rnn.initHidden() 
        for t in range(len(seq)-1):
            x = Variable(torch.LongTensor([seq[t]]).unsqueeze(0))
            y = Variable(torch.LongTensor([seq[t+1]]))
            output,hidden = rnn(x,hidden)
            loss += criterion(output,y)
        loss = 1.0 * loss / len(seq)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss
        if i>0 and i % 500 ==0:
            print('第{}轮，第{}个，训练平均loss：{:.2f}'.format(epoch,i,train_loss.data.numpy()/i))
    valid_loss = 0
    errors = 0
    show_out_p =''
    show_out_t = ''
    for i,seq in enumerate(validset):
        loss = 0
        outstring = ''
        targets = ''
        diff = 0
        hidden = rnn.initHidden()
        for t in range(len(seq)-1):
            x = Variable(torch.LongTensor([seq[t]]).unsqueeze(0))
            y = Variable(torch.LongTensor([seq[t+1]]))
            output,hidden = rnn(x,hidden)
            data = output.data.numpy()
            print("the output is ",data)
            mm = torch.max(output,1)[1][0]
            outstring += str(mm.data.numpy())
            targets += str(y.data.numpy()[0])
            loss += criterion(output,y)
            diff += 1 - mm.eq(y).data.numpy()[0]
        loss = 1.0 * loss / len(seq)
        valid_loss += loss
        errors += diff
        if np.random.rand() < 0.1:
            show_out_p += outstring 
            show_out_t += targets
        print(output[0][2].data.numpy())
        print('第{}轮，训练loss: {:.2f},校验loss：{:.2f},错误率：{:.2f}'.format(epoch,train_loss.data.numpy()/len(train_set),                                                                               valid_loss.data.numpy()/len(validset)                                                                                   ,1.0*errors/len(validset)))
        print("the show output is: ",show_out_p)
        print("the show taget is: ",show_out_t)
        results.append([train_loss.data.numpy()/len(train_set),valid_loss/len(train_set),1.0*errors/len(validset)])

import numpy
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils

raw_text = open(r'F:南昌大学自然语言处理学习自然语言处理资料课件资料6DLinNLPinputWinston_Churchil.txt', encoding='utf-8').read()
raw_text = raw_text.lower()

chars = sorted(list(set(raw_text)))
char_to_int = dict((c,i) for i,c in enumerate(chars))
int_to_char = dict((i,c) for i,c in enumerate(chars))

len(chars)

len(raw_text)

seq_length = 100
x = []
y = []
for i in range(0, len(raw_text) - seq_length):
    given = raw_text[i:i+seq_length]
    predict = raw_text[i + seq_length]
    x.append([char_to_int[char] for char in given])
    y.append(char_to_int[predict])
print(x[:3])
print(y[:3])

n_patterns = len(x)
n_vocab = len(chars)
x = numpy.reshape(x, (n_patterns, seq_length, 1))
x = x/float(n_vocab)
y = np_utils.to_categorical(y)
print(x[11])
print(y[11])

model = Sequential()
model.add(LSTM(256, input_shape=(x.shape[1], x.shape[2])))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.fit(x, y, nb_epoch=50, batch_size=4096)

def predict_next(input_array):
    x = numpy.reshape(input_array, (1, seq_length, 1))
    x = x / float(n_vocab)
    y = model.predict(x)
    return y

def string_to_index(raw_input):
    res = []
    for c in raw_input[(len(raw_input)-seq_length):]:
        res.append(char_to_int[c])
    return res

def y_to_char(y):
    largest_index = y.argmax()
    c = int_to_char[largest_index]
    return c

def generate_article(init, rounds=200):
    in_string = init.lower()
    for i in range(rounds):
        n = y_to_char(predict_next(string_to_index(in_string)))
        in_string += n
    return in_string

init = 'His object in coming to New York was to engage officers for that service. He came at an opportune moment'
article = generate_article(init)
print(article)

import os
import numpy as np
import nltk
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
from gensim.models.word2vec import Word2Vec

raw_text = ''
for file in os.listdir(r'C:\Users\Administrator\NLP\复习八天学习自然语言处理内容\第六天input\'):
    if file.endswith(".txt"):
        raw_text += open(r'C:\Users\Administrator\NLP\复习八天学习自然语言处理内容\第六天input\'+file, errors='ignore').read() + 'nn'
# raw_text = open('../input/Winston_Churchil.txt').read()
raw_text = raw_text.lower()
sentensor = nltk.data.load('tokenizers/punkt/english.pickle')        
sents = sentensor.tokenize(raw_text)
corpus = []
for sen in sents:
    corpus.append(nltk.word_tokenize(sen))

print(len(corpus))
print(corpus[:3])

w2v_model = Word2Vec(corpus, size=128, window=5, min_count=5, workers=4)
w2v_model['office']

raw_input = [item for sublist in corpus for item in sublist]
len(raw_input)
raw_input[12]
text_stream = []
vocab = w2v_model.wv.vocab
for word in raw_input:
    if word in vocab:
        text_stream.append(word)
len(text_stream)

seq_length = 10
x = []
y = []
for i in range(0, len(text_stream) - seq_length):

    given = text_stream[i:i + seq_length]
    predict = text_stream[i + seq_length]
    x.append(np.array([w2v_model[word] for word in given]))
    y.append(w2v_model[predict])
print(x[10])
print(y[10])
print(len(x))
print(len(y))
print(len(x[12]))
print(len(x[12][0]))
print(len(y[12]))
print(x[10])
print(y[10])
print(len(x))
print(len(y))
print(len(x[12]))
print(len(x[12][0]))
print(len(y[12]))

x = np.reshape(x, (-1, seq_length, 128))
y = np.reshape(y, (-1,128))

model = Sequential()
model.add(LSTM(256, dropout_W=0.2, dropout_U=0.2, input_shape=(seq_length, 128)))
model.add(Dropout(0.2))
model.add(Dense(128, activation='sigmoid'))
model.compile(loss='mse', optimizer='adam')
model.fit(x, y, nb_epoch=50, batch_size=4096)

def predict_next(input_array):
    x = np.reshape(input_array, (-1,seq_length,128))
    y = model.predict(x)
    return y

def string_to_index(raw_input):
    raw_input = raw_input.lower()
    input_stream = nltk.word_tokenize(raw_input)
    res = []
    for word in input_stream[(len(input_stream)-seq_length):]:
        res.append(w2v_model[word])
    return res

def y_to_word(y):
    word = w2v_model.most_similar(positive=y, topn=1)
    return word
def generate_article(init, rounds=30):
    in_string = init.lower()
    for i in range(rounds):
        n = y_to_word(predict_next(string_to_index(in_string)))
        in_string += ' ' + n[0][0]
    return in_string
init = 'Language Models allow us to measure how likely a sentence is, which is an important for Machine'
article = generate_article(init)
print(article)

import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score
from datetime import date
data = pd.read_csv(r'F:南昌大学自然语言处理学习自然语言处理资料课件资料6DLinNLPinputCombined_News_DJIA.csv')
data.head()

train = data[data['Date'] < '2015-01-01']
test = data[data['Date'] > '2014-12-31']

X_train = train[train.columns[2:]]
corpus = X_train.values.flatten().astype(str)

X_train = X_train.values.astype(str)
X_train = np.array([' '.join(x) for x in X_train])
X_test = test[test.columns[2:]]
X_test = X_test.values.astype(str)
X_test = np.array([' '.join(x) for x in X_test])
y_train = train['Label'].values
y_test = test['Label'].values

corpus[:3]
X_train[:1]
y_train[:5]
from nltk.tokenize import word_tokenize
corpus = [word_tokenize(x) for x in corpus]
X_train = [word_tokenize(x) for x in X_train]
X_test = [word_tokenize(x) for x in X_test]
X_train[:2]
corpus[:2]

from nltk.corpus import stopwords
stop = stopwords.words('english')
import re
def hasNumbers(inputString):
    return bool(re.search(r'd', inputString))
def isSymbol(inputString):
    return bool(re.match(r'[^w]', inputString))
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
def check(word):
    word= word.lower()
    if word in stop:
        return False
    elif hasNumbers(word) or isSymbol(word):
        return False
    else:
        return True
def preprocessing(sen):
    res = []
    for word in sen:
        if check(word):
            word = word.lower().replace("b'", '').replace('b"', '').replace('"', '').replace("'", '')
            res.append(wordnet_lemmatizer.lemmatize(word))
    return res

corpus = [preprocessing(x) for x in corpus]
X_train = [preprocessing(x) for x in X_train]
X_test = [preprocessing(x) for x in X_test]
print(corpus[553])
print(X_train[523])

from gensim.models.word2vec import Word2Vec
model = Word2Vec(corpus, size=128, window=5, min_count=5, workers=4)
model['ok']

vocab = model.wv.vocab
def get_vector(word_list):
    res = np.zeros([128])
    count = 0
    for word in word_list:
        if word in vocab:
            res += model[word]
            count += 1
    return res/count
get_vector(['hello', 'from', 'the', 'other', 'side'])

wordlist_train = X_train
wordlist_test = X_test
X_train = [get_vector(x) for x in X_train]
X_test = [get_vector(x) for x in X_test]
print(X_train[10])

from sklearn.svm import SVR
from sklearn.model_selection import cross_val_score

params = [0.1,0.5,1,3,5,7,10,12,16,20,25,30,35,40]
test_scores = []
for param in params:
    clf = SVR(gamma=param)
    test_score = cross_val_score(clf, X_train, y_train, cv=3, scoring='roc_auc')
    test_scores.append(np.mean(test_score))
import matplotlib.pyplot as plt
%matplotlib inline
plt.plot(params, test_scores)
plt.title("Param vs CV AUC Score");