使用fasttext实现文本处理及文本预测

#encoding=utf-8
import jieba
#author linxinzhu
seg_list = jieba.cut("这个竞赛真的费时间",cut_all=True)
print "Full Mode:", "/ ".join(seg_list) #全模式

seg_list = jieba.cut("zwq沉迷逛QQ空间，还时不时撩妹",cut_all=False)
print "Default Mode:", "/ ".join(seg_list) #精确模式

seg_list = jieba.cut("测试集好大啊，跑一次要好久") #默认是精确模式
print ", ".join(seg_list)

seg_list = jieba.cut_for_search("这篇博客是在2017年11月17日写的，
各位看官觉得有用的话，可以评论点赞") #搜索引擎模式
print ", ".join(seg_list)

PS C:UsersLinXinzhupy> python fenci.py
Full Mode:Building prefix dict from C:Python27libsite-packagesjiebadict.txt ...
Loading model from cache c:userslinxin~1appdatalocaltempjieba.cache
Loading model cost 0.804000139236 seconds.
Prefix dict has been built succesfully.
 这个/ 竞赛/ 真的/ 费时/ 费时间/ 时间
Default Mode: zwq/ 沉迷/ 逛/ QQ/ 空间/ ，/ 还/ 时不时/ 撩妹
测试, 集好, 大, 啊, ，, 跑, 一次, 要, 好久
这篇, 博客, 是, 在, 2017, 年, 11, 月, 17, 日写, 的, ，, 各位, 看官, 觉得, 有用, 的话, ，, 可以, 评论, 点赞

def go_split(s,min_len):
    # 拼接正则表达式
    symbol = '，；。！、？!'
    symbol = "[" + symbol + "]+"
    # 一次性分割字符串
    result = re.split(symbol, s)
    return [x for x in result if len(x)>min_len]

def is_dup(s,min_len):
    result = go_split(s,min_len)
    return len(result) !=len(set(result))

def is_neg_symbol(uchar):
    neg_symbol=['!', '0', ';', '?', '、', '。', '，']
    return uchar in neg_symbol

if (ur"，的" in s0) and (not(ur"，的确" in s0)) and (not(ur"，的士" in s0)) 
                        and (not(ur"，的哥" in s0)) and (not(ur"，的的确确" in s0)):
        flag = "NEGATIVE"
    if (ur"，了" in s0) and (not(ur"，了解" in s0)) and (not(ur"，了结" in s0)) 
                        and (not(ur"，了无" in s0)) and (not(ur"，了却" in s0)) 
                        and (not(ur"，了不起" in s0)):
        flag = "NEGATIVE"

    if (ur"。的" in s0) and (not(ur"。的确" in s0)) and (not(ur"。的士" in s0)) 
                        and (not(ur"。的哥" in s0)) and (not(ur"。的的确确" in s0)):
        flag = "NEGATIVE"
    if (ur"。了" in s0) and (not(ur"。了解" in s0)) and (not(ur"。了结" in s0)) 
                        and (not(ur"。了无" in s0)) and (not(ur"。了却" in s0)) 
                        and (not(ur"。了不起" in s0)):
        flag = "NEGATIVE"

    if (ur";的" in s0) and (not(ur";的确" in s0)) and (not(ur";的士" in s0)) 
                        and (not(ur";的哥" in s0)) and (not(ur";的的确确" in s0)):
        flag = "NEGATIVE"
    if (ur";了" in s0) and (not(ur"了解;" in s0)) and (not(ur";了结" in s0)) 
                        and (not(ur";了无" in s0)) and (not(ur";了却" in s0)) 
                        and (not(ur";了不起" in s0)):
        flag = "NEGATIVE"

    if (ur"?的" in s0) and (not(ur"?的确" in s0)) and (not(ur"?的士" in s0)) 
                        and (not(ur"?的哥" in s0)) and (not(ur"?的的确确" in s0)):
        flag = "NEGATIVE"
    if (ur"?了" in s0) and (not(ur"?了解" in s0)) and (not(ur"?了结" in s0)) 
                        and (not(ur"?了无" in s0)) and (not(ur"?了却" in s0)) 
                        and (not(ur"?了不起" in s0)):
        flag = "NEGATIVE"

#encoding=utf-8
#author linxinzhu
import jieba
import sys
reload(sys)
sys.setdefaultencoding('utf8')
i = 0
count=0
f = open("train.tsv", 'r')
#f = open("evaluation_public.tsv", 'r')
outf = open("lab3fenci.csv",'w')
#outf = open("lab3fencitest.csv",'w')

for line in f:
    r = ""
    try:
        r = line.decode("UTF-8")
    except:
        print "charactor code error UTF-8"
        pass
    if r == "":
        try:
            r = line.decode("GBK")
        except:
            print "charactor code error GBK"
            pass
    line=line.strip()
    l_ar=line.split("t")
    if len(l_ar)!=4:
        continue
    id=l_ar[0]
    title=l_ar[1]
    content=l_ar[2]
    lable=l_ar[3]

    seg_title=jieba.cut(title.replace("t"," ").replace("n"," "))
    seg_content=jieba.cut(content.replace("t"," ").replace("n"," "))
    #r=" ".join(seg_title)+" "+" ".join(seg_content)+"n"
    outline = " ".join(seg_title)+"t"+" ".join(seg_content)
    outline = "t__label__" + lable + outline+"t"
    outf.write(outline)

    if i%2500 == 0:
        count=count+1
        sys.stdout.flush()
        sys.stdout.write("#")
    i=i+1

f.close()
outf.close()
print "nWord segmentation complete."
print i

pip install fasttext

import fasttext

# Skipgram model
model = fasttext.skipgram('data.txt', 'model')
print model.words # list of words in dictionary

# CBOW model
model = fasttext.cbow('data.txt', 'model')
print model.words # list of words in dictionary

classifier = fasttext.supervised('data.train.txt', 'model')

result = classifier.test('test.txt')
print 'P@1:', result.precision
print 'R@1:', result.recall
print 'Number of examples:', result.nexamples

texts = ['example very long text 1', 'example very longtext 2']
labels = classifier.predict(texts)
print labels

# Or with the probability
labels = classifier.predict_proba(texts)
print labels

input_file     training file path (required)
output         output file path (required)
lr             learning rate [0.05]
lr_update_rate change the rate of updates for the learning rate [100]
dim            size of word vectors [100]
ws             size of the context window [5]
epoch          number of epochs [5]
min_count      minimal number of word occurences [5]
neg            number of negatives sampled [5]
word_ngrams    max length of word ngram [1]
loss           loss function {ns, hs, softmax} [ns]
bucket         number of buckets [2000000]
minn           min length of char ngram [3]
maxn           max length of char ngram [6]
thread         number of threads [12]
t              sampling threshold [0.0001]
silent         disable the log output from the C++ extension [1]
encoding       specify input_file encoding [utf-8]

model = fasttext.skipgram('train.txt', 'model', lr=0.1, dim=300)

classifier.labels                  # List of labels
classifier.label_prefix            # Prefix of the label
classifier.dim                     # Size of word vector
classifier.ws                      # Size of context window
classifier.epoch                   # Number of epochs
classifier.min_count               # Minimal number of word occurences
classifier.neg                     # Number of negative sampled
classifier.word_ngrams             # Max length of word ngram
classifier.loss_name               # Loss function name
classifier.bucket                  # Number of buckets
classifier.minn                    # Min length of char ngram
classifier.maxn                    # Max length of char ngram
classifier.lr_update_rate          # Rate of updates for the learning rate
classifier.t                       # Value of sampling threshold
classifier.encoding                # Encoding that used by classifier
classifier.test(filename, k)       # Test the classifier
classifier.predict(texts, k)       # Predict the most likely label
classifier.predict_proba(texts, k) # Predict the most likely label include their probability

classifier = fasttext.supervised("lab3fenci.csv","lab3fenci.model",

result = classifier.test("lab3fenci.csv")

texts = ['它被誉为"天下第一果"，补益气血，养阴生津，现在吃正应季!  六七月是桃
子大量上市的季节，因其色泽红润，肉质鲜美，有个在实验基地里接受治疗的妹子。广受大
众的喜爱。但也许你并不知道，看惯了好莱坞大片眼花缭乱的特效和场景。它的营养也是很
高的，不仅富含多种维生素、矿物质及果酸，至少他们一起完成了一部电影，其含铁量亦居
水果之冠，被誉为"天下第一果"。1、在来世那个平行世界的自己。增加食欲，养阴生津的
作用，可用于大病之后，气血亏虚，面黄肌瘦，Will在海滩上救下了Isla差点溺水的儿
子。心悸气短者。2、最近有一部叫做《爱有来世》的科幻电影。桃的含铁量较高，就越容
易发现事情的真相。是缺铁性贫血病人的理想辅助食物。3、桃含钾多，含钠少，适合水肿
病人食用。4、桃仁有活血化淤，润肠通作用，可用于闭经、跌打损伤等辅助治疗。胃肠功
能弱者不宜吃桃、桃仁提取物有抗凝血作用，而Will也好像陷入魔怔一般。并能抑制咳嗽中
枢而止咳，扩展"科学来自于人性"的概念。同时能使血压下降，片中融合了很多哲学、宗教
的玄妙概念，可用于高血压病人的辅助治疗。6、桃花有消肿、利尿之效，可用于治疗浮肿
腹s水，大便干结，小便不利和脚气足肿。一段美好的故事才就此开始。桃子性热，味甘
酸，具有补心、解渴、不过都十分注重内核的表达，充饥、生津的功效，父亲没有继续在房
间埋头工作。']
    labels = classifier.predict(li)    
    print labels

classifier = fasttext.supervised("lab3fenci.csv","lab3fenci.model",
label_prefix="__label__",lr=0.1,epoch=100,dim=200,bucket=5000000)
result = classifier.test("lab3fenci.csv")
print result.precision
print result.recall

classifier = fasttext.load_model('lab3fenci.model.bin', label_prefix='__label__')

# _*_coding:utf-8 _*_
import fasttext
#author linxinzhu
#load训练好的模型
classifier = fasttext.load_model('lab3fenci.model.bin',
 label_prefix='__label__')

i=0
f = open("evaluation_public.tsv", 'r')
outf = open("sub.csv",'w')
for line in f:
    outline=""
    if i==400000:
         break
    r = ""
    try:
        r = line.decode("UTF-8")
    except:
        print "charactor code error UTF-8"
        pass
    if r == "":
        try:
            r = line.decode("GBK")
        except:
            print "charactor code error GBK"
            pass
    line=line.strip()
    l_ar=line.split("t")
    id=l_ar[0]
    title=l_ar[1]
    content=l_ar[2]
    s=""
    li=[]
    li=list()
    s="".join(content)
    li=s.split("$$$$$$$$")  
    texts=li 
    labels = classifier.predict(li)
    #print id
    #print labels
    strlabel=str(labels)
    if strlabel=="POSITIVE" :
        outline = id+"," + "POSITIVE" + "n"
        outf.write(outline)
    if strlabel=="NEGATIVE" :
        outline = id+"," + "NEGATIVE" + "n"
        outf.write(outline)
    i=i+1
    del s
    del li
f.close()
outf.close()