TF使用例子-情感分类

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
import re
import time
from bs4 import BeautifulSoup
req_header = {   
 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',   
 'Accept-Language':'zh - CN, zh;q = 0.8, en;q = 0.6',    
'Connection':'keep - alive',    
'Cookie':'_hc.v = ""4e10f82f-cdb5-4fa1-ba89-262a394be3d1.1490604667""; PHOENIX_ID = 0a01084a - 15b1299fd35 - 163c9ff; s_ViewType = 10; JSESSIONID = F255BB7A08A17AFC8F2E3701599B3193; aburl = 1; cy = 6; cye = suzhou',   
 'Upgrade-Insecure-Requests':'1',   
 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}
 history = {} #记录链接是否已经爬取过，防止重定向
for subjectid in range(1,220000):     
pageid = 1     
cur_url = "http://bgm.tv/subject/" + str(subjectid)+ "/comments"     
while(True):         

mark = 0         
cur_url = cur_url + "?page=" + str(pageid)        
if cur_url in history: # 是否爬取过             
break         
else:             
history[cur_url
 ![@H95LRLI39{5M`FA3OIY{QD.png](http://upload-images.jianshu.io/upload_images/77013-6993ddc74df9c560.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240)] = 1        
 try:             
r = requests.get(cur_url, headers=req_header, timeout=10)         
except:            
print(subjectid)             
time.sleep(5)           
 try:                 
r = requests.get(cur_url, headers=req_header, timeout=10)             
except:                
break 
#爬两次还爬不到就放弃换下一个         
content = BeautifulSoup(r.content, 'lxml')         
base_url_index = r.url.find("?")        
 cur_url = r.url[0:base_url_index]        
try:            

 title = content.find("a",href=re.compile(r"/subject*")).text         
except:            
break        
 if title == "": #爬取到的页面有问题，比如404             
break        
 # 因为把每个词条都作为文件名，所以有些特殊字符不能作为文件名        
 title = title.replace("/","-").replace(":","-").replace("""," ").replace("<", "(").replace(">", ")").replace("?","-").replace("*","-").replace("\","-").replace("|","-").replace("n","-")                  
 # 把爬取到的条目下的评论都拿出来放到条目文件里        
 with open("directory/"+title, "a", encoding="utf-8")as f:           
 for item in content.find("div", {"id": "comment_box"}).find_all("div", {"class":"item clearit"}):                 item = item.find("div",{"class": re.compile(r"text_main_*")})               
 try:                    
print(item.find("span")["class"])                 
except:                   
break                 
f.write(item.find("span")["class"][0] + 'n')                 
f.write(item.text.strip())                 
mark = 1                
 f.write("n===========================n")        
 pageid += 1         
if mark == 0:            
break
time.sleep(0.3) # 睡一会，别爬太快

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import osimport re
out = open("bangumi_sentiment", "w", encoding="utf-8")for cur_file in os.listdir("directory"):    
with open("directory/"+cur_file, "r", encoding="utf-8") as f:
        comments = f.read().split("===========================")        
for comment in comments:           
 if comment == "":                
continue
            start_index = comment.find("sstars")
            score = comment[start_index+6:8].strip()            
try:               
 #找到日期末尾位置
                content_index = re.search("d{4}-d{1,2}-d{1,2} d{1,2}:d{1,2}  ",comment).end() 
except:                
continue
out.write(score+" ")
out.write(" ".join(comment[content_index:].split("n"))+"n")

#!/usr/bin/env python# 
-*- coding: utf-8 -*-
import logging
import multiprocessing
import sysfrom gensim.models 
import Word2Vec
from gensim.models.word2vec 
import LineSentencedef train(inp, model_name, size=100, word_frequency_threshold=5):
    logging.basicConfig(format = '%(asctime)s : %(levelname)s : %(message)s', level = logging.INFO) 
    print("Begin to train model ...")
    model = Word2Vec(LineSentence(inp),
    size = size,
    window = 10,
    hs = 1,
    sg = 1,  # skip-gram
    min_count = word_frequency_threshold,
    workers = multiprocessing.cpu_count()/2)  #CPU数量
    model.save_word2vec_format(model_name, binary=False) # save a model in order to be checkedif __name__ == "__main__":    if len(sys.argv) <3:
        print("please input input file and model file")
    inp = sys.argv[1]
    model = sys.argv[2]
    word_size = 100
    word_threshold = 3
    if len(sys.argv) > 3:
        word_size = int(sys.argv[3]) #词向量维度
        word_threshold = int(sys.argv[4]) #每个字最少出现的次数
    train(inp, model, word_size, word_threshold)

# 设置placeholderword_ids = tf.placeholder(tf.int32, shape=[None, None],name="word_ids") 
 # batch size, max length of sentence in batchsequence_lengths = tf.placeholder(tf.int32, shape=[None], name="sequence_length")  
# shape = batch sizelabels = tf.placeholder(tf.int32, shape=[None], name="labels")  
# only one dimensiondropout = tf.placeholder(dtype=tf.float32, shape=[], name="dropout")
lr = tf.placeholder(dtype=tf.float32, shape=[], name="lr")

# 设置embedding层with tf.variable_scope("words"):
    _word_embeddings = tf.Variable(embeddings, name="_word_embeddings", dtype=tf.float32,
                                   trainable=trainable)
    word_embeddings = tf.nn.embedding_lookup(_word_embeddings, word_ids,
                                             name="word_embeddings")
    word_embeddings = tf.nn.dropout(word_embeddings, dropout)# 设置模型with tf.variable_scope("bi-lstm"):
    lstm_cell = tf.contrib.rnn.LSTMCell(hidden_size)
    _, (output_state_fw, output_state_bw) = tf.nn.bidirectional_dynamic_rnn(lstm_cell,
     lstm_cell, word_embeddings,
     sequence_length=sequence_lengths,
     dtype=tf.float32)
    output = tf.concat((output_state_fw[-1], output_state_bw[-1]), axis=-1)
    output = tf.nn.dropout(output, dropout)
# 输出部分在双向lstm的最后一层合并后加一个全连接层，全连接层后接一个softmax层with tf.variable_scope("proj"):
    W = tf.get_variable("W", shape=[2 * hidden_size, nlabels],
                        dtype=tf.float32)
    b = tf.get_variable("b", shape=[nlabels], dtype=tf.float32,
                        initializer=tf.zeros_initializer())
    output = tf.reshape(output, [-1, 2 * hidden_size])
    pred = tf.matmul(output, W) + b
    logits = pred
labels_pred = tf.cast(tf.argmax(logits, axis=-1), tf.int32)
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)
loss = tf.reduce_mean(losses) # batch的平均losswith tf.variable_scope("train_step"):
    optimizer = tf.train.AdamOptimizer(lr)
    train_op = optimizer.minimize(loss)

# 分类模型def create_model_classify(max_features, nlabels, embeddings=None, embedding_dim=200, hidden_dim=300):
    model = Sequential()    if embeddings is not None:
        model.add(Embedding(input_dim=max_features,output_dim=embedding_dim, dropout=0.2, weights=[embeddings], mask_zero=True))    else:
        model.add(Embedding(max_features, embedding_dim=embedding_dim, dropout=0.2))
    model.add(Bidirectional(LSTM(hidden_dim, dropout_W=0.2, dropout_U=0.2, return_sequences=False)))  # try using a GRU instead, for fun
    model.add(Dense(nlabels))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy',]) # availabel metrics https://keras.io/metrics/
    return model# 回归模型def create_model_regress(max_features, embeddings=None, embedding_dim=200, hidden_dim=300):
    model = Sequential()    if embeddings is not None:
        model.add(Embedding(input_dim=max_features,output_dim=embedding_dim, dropout=0.2, weights=[embeddings], mask_zero=True))    else:
        model.add(Embedding(max_features, embedding_dim=embedding_dim, dropout=0.2))
    model.add(
        Bidirectional(LSTM(hidden_dim, dropout_W=0.2, dropout_U=0.2),
                      merge_mode='concat'))  

    model.compile(loss='mean_squared_error',
                  optimizer='sgd',
                  metrics=['mae','acc' ])  # availabel metrics https://keras.io/metrics/idden_dim=
return model

#keras实现from keras.models import Model
org_model = load_model("result/model.weights/acc_XXX.model")
layer_name = 'bidirectional_1'intermediate_layer_model = Model(input=org_model.input,   
# keras 2.0  inputs  outputs
output=org_model.get_layer(layer_name).output)
intermediate_output = intermediate_layer_model.predict(np.asarray(word_ids))