【实战帖】使用Python分析社交网络数据

easy_install beautifulsoup4

import urllib2from bs4 import BeautifulSoup#设置默认encoding方式import sysreload(sys)sys.setdefaultencoding('gbk')url = 'http://news.baidu.com/' #待抓取的网页地址content = urllib2.urlopen(url).read() #获取网页的html文本#使用BeautifulSoup解析htmlsoup = BeautifulSoup(content, from_encoding = 'gbk') #识别热点新闻hotNews = soup.find_all('div', {'class', 'hotnews'})[0].find_all('li')for i in hotNews:
    print i.a.text #打印新闻标题
    print i.a['href'] #打印新闻链接

习近平：改革惟其艰难 才更显勇毅http://china.cankaoxiaoxi.com/2014/0808/454139.shtml治疗费政府全担http://gb.cri.cn/42071/2014/08/06/2165s4643613.htm李克强点赞人物盘点：有女县长也有棒棒军http://news.youth.cn/jsxw/201408/t20140808_5605703.htm李克强指示地震救灾http://yn.yunnan.cn/html/2014-08/07/content_3316250.htm云南地震致615人遇难http://news.baidu.com/z/ynlddz/new/zhuanti.html临时安置点1顶帐篷内住20人http://news.youth.cn/gn/201408/t20140808_5607225.htm

easy_install sinaweibopy

mid = client.get.statuses__queryid(mid = 'Bhd8k0Jv8', isBase62 = 1, type = 1)['id']

client = weiboClient() #连接到API接口#定义存储文档地址dataFile = open("D:/github/weibo_repost/weibo_repost_all.csv",'wb') start = clock() #定义初始时间#使用for循环遍历所有的待抓取页面for page in range(1, pageNum + 1):
    getReposts(mid, page) # 抓取单页的转发信息dataFile.close() # 关闭存储文件

#定义二度转发的函数def get2stepReposts(mid, uid, page):    #将被二度转发的微博的mid和uid传递进来
    try:
        r = client.get.statuses__repost_timeline(id = mid, page = page, count = 200)
        if len(r) == 0:
            pass
        else:
            m = int(len(r['reposts']))
        for i in range(0, m):
            """1.1 转发微博的属性"""
            rt_mid = r['reposts'][i].id
            created = r['reposts'][i].created_at
            """1.2 微博转发者的属性"""
            user = r['reposts'][i].user
            rt_uid = user.id
            print >>dataFile2, "%s,%s,%s,%s,'%s'"  % (mid, uid, rt_mid, rt_uid, created)   
    except Exception, e:
        print >> sys.stderr, 'Encountered Exception:', e, page
        time.sleep(120)
        pass 

with open("D:/github/weibo_repost/weibo_repost_all.csv") as f:
    for line in f:
        line = line.strip().split(',')
        repostCount = int(line[3])
        mid2step = line[0]
        uid = line[7]
        if repostCount > 0:
            print mid2step, repostCount            
            if repostCount%200 == 0:
                pages = repostCount/200
            else:
                pages = int(repostCount/200) + 1
            dataFile2 = open("D:/github/weibo_repost/weibo_repost_2_step.csv",'a') 
            for page in range(1, pages + 1):
                get2stepReposts(mid2step, uid, page)
            dataFile2.close()  

#首先得到所有的转发者列表with open("D:/github/weibo_repost/weibo_repost_all.csv") as f:
    mids = []    
    for line in f:
        line = line.strip().split(',')
        mid = line[0]
        mids.append(mid)#然后获取二步转发网络中的信息流入节点          with open("D:/github/weibo_repost/weibo_repost_2_step.csv") as f:
    toNodes = []
    for line in f:
        line = line.strip().split(',')
        mid = line[2]
        toNodes.append(mid)#找到未出现在二步转发网络中的信息流入节点的转发者       oneStep = []        for i in mids:
    if i not in toNodes:
        oneStep.append(i)#构建一步转发数据with open("D:/github/weibo_repost/weibo_repost_all.csv") as f:   
    for line in f:
        line = line.strip().split(',')
        created = line[1]
        mid = line[0]
        uid = line[7]
        rtmid = line[18]
        rtuid= line[20]
        if mid in oneStep:
            with open('D:/github/weibo_repost/weibo_repost_1_step.csv', 'a') as g:
                record = rtmid + ','+ rtuid+','+ mid + ','+uid+','+created
                g.write(record+"n")#合并一步转发和二步转发，得到完整的转发网络。with open("D:/github/weibo_repost/weibo_repost_1_step.csv") as f:
    for line in f:
        with open('D:/github/weibo_repost/weibo_reposts_network.csv', 'a') as g:
            g.write(line)with open("D:/github/weibo_repost/weibo_repost_2_step.csv") as f:
    for line in f:
        with open('D:/github/weibo_repost/weibo_reposts_network.csv', 'a') as g:
            g.write(line)

# 首先，提取转发时间信息with open('D:/github/weibo_repost/weibo_repost_network.csv', 'r') as f:
    rt_time = []
    for line in f:
        time= line.strip().split(',')[-1]
        day = time[7:9]
        hms= time[9:17].replace(':', '')
        time = int(day + hms)
        rt_time.append(time)
 #计算转发时间的先后顺序import numpyarray = numpy.array(rt_time)order = array.argsort()ranks = order.argsort()

# 构建微博转发网络  import matplotlib.pyplot as pltimport networkx as nx      G = nx.Graph()with open('D:/github/weibo_repost/weibo_repost_network.csv', 'r') as f:
    for position, line in enumerate(f):
        mid, uid, rtmid, rtuid= line.strip().split(',')[:-1]
        G.add_edge(uid, rtuid, time = ranks[position])edges,colors = zip(*nx.get_edge_attributes(G,'time').items())degree=G.out_degree()#计算节点的出度path_length = nx.all_pairs_shortest_path_length(G)depth =[ path_length['2803301701'][i] for i in degree.keys()]pos=nx.spring_layout(G) #设置网络的布局fig = plt.figure(figsize=(10, 8),facecolor='white')nx.draw(G, pos, nodelist = degree.keys(),
        node_size = [np.sqrt(v+1)*10 for v in (degree.values())], node_color = depth, 
        node_shape = 'o', cmap=plt.cm.gray,
        edgelist = edges, edge_color = 'gray', width = 0.5,
        with_labels = False, arrows = False)plt.show()

UG=G.to_undirected()eccen = nx.eccentricity(UG)#节点离心度max(eccen.values() #4min(eccen.values() #2nx.diameter(G) # 网络直径4nx.radius(G) #网络半径2path = nx.all_pairs_shortest_path(G)nx.shortest_path(G, source = '2803301701', target ='1783628693' )#['2803301701', '1904107133', '1783628693']nx.shortest_path_length(G, source = '2803301701', target ='1783628693' ) #2nx.average_shortest_path_length(G) # 网络平均最短距离0.001nx.average_shortest_path_length(UG) # 网络平均最短距离2.05

degree = nx.degree(G)closenesss = nx.closeness_centrality(G)betweenness = nx.betweenness_centrality(G)

degree_hist = nx.degree_histogram(G)x = range(len(degree_hist))y = [i / int(sum(degree_hist)) for i in degree_hist]plt.loglog(x, y, color = 'blue', linewidth = 2, marker = 'o')plt.title('Degree Distribution')plt.ylabel('Probability')plt.xlabel('Degree')plt.show()

# 绘制网络度排名的概率分布图from collections import defaultdictimport scipy.stats as ssd = sorted(degree.values(), reverse = True )d_table = defaultdict(int)for k in d:
    d_table[k] += 1d_value = sorted(d_table)d_freq = [d_table[i] for i in d_value]d_prob = [i/sum(d_freq) for i in d_freq]d_rank = ss.rankdata(d_value).astype(int)plt.loglog(d_rank, d_prob, color = 'blue', linewidth = 2, marker = 'o')plt.title('Degree Rank-order Distribution')plt.ylabel('Probability')plt.xlabel('Rank')plt.show()

G.number_of_nodes() # 节点数量1047 G.number_of_edges() # 链接数量1508nx.density(G) # 网络密度0.001nx.info(G)#'Name: nType: DiGraphnNumber of nodes: 1047nNumber of edges: 1518nAverage in degree:   1.4499nAverage out degree:   1.4499'nx.transitivity(G) # 传递性0.001nx.average_clustering(UG) # 网络群聚系数0.227nx.degree_assortativity_coefficient(UG) # 匹配性-0.668

eccen = nx.eccentricity(UG) #节点离心度eccen['2803301701']  #深度为2, '2803301701'即@人民日报#找到扩散深度最大的节点path_length = nx.all_pairs_shortest_path_length(G)oPath = path_length['2803301701']maxDepth = filter(lambda x: oPath[x] == max(oPath.values()), oPath.keys())len(maxDepth) # 扩散深度最大的节点数量29

from datetime import datetimewith open('D:/github/weibo_repost/weibo_repost_network.csv', 'r') as f:
    rt_time = []
    for line in f:
        time= line.strip().split(',')[-1]
        day = time[7:9]
        hms= time[9:17].replace(':', '')
        time = int(day + hms)
        rt_time.append(time)day = [('2014-08-0'+str(i)[:1]+'-'+str(i)[1:3]) for i in rt_time]day = [datetime.strptime(d, '%Y-%m-%d-%H') for d in day]day_weibo = datetime.strptime('2014-08-07-19', '%Y-%m-%d-%H') #源微博发出时间hours = [(i-day_weibo).total_seconds()/3600 for i in day]values, base = np.histogram(hours, bins = 40)cumulative = np.cumsum(values)plt.subplot(1, 2, 1)plt.plot(base[:-1], cumulative, c = 'red')plt.title('Cumulative Diffusion')plt.ylabel('Number of Retweets')plt.xlabel('Hours')plt.subplot(1, 2, 2 )plt.plot(base[:-1], values, c = 'orange')plt.title('Hourly Diffusion')plt.xlabel('Hours')plt.show()

from collections import defaultdictfrom matplotlib.font_manager import FontPropertiesimport numpy as np#提取微博转发者的省份信息with open("D:/github/weibo_repost/weibo_repost_all.csv") as f:  
    province = dict()
    for line in f:
        line = line.strip().split(',')
        uid = line[7]
        province[uid] = line[9]#读取省份代码列表with open("D:/github/weibo_repost/weibo_province.txt") as f: 
    province_dict = dict()    
    for line in f:
        pid = line.split('t')[0]
        provinceName = line.split('t')[1].split(',')[0]
        province_dict[pid] = provinceName#获取省份名称        province_info = [province_dict[i] for i in province.values()]#获取各个省份的微博转发数量province_table = defaultdict(int)for k in province_info:
    province_table[k] += 1#根据微博转发数量对省份名称排序   province_table_sorted = sorted([(value, key) for (key, value) in province_table.items()])#获取排序后的省份微博转发数量和对应的省份名称    provinceFreq= [i[0] for i in province_table_sorted]provinceName = [unicode(i[1]) for i in province_table_sorted]#绘制直方图position = np.arange(len(provinceName))ax = plt.axes()ax.set_xticks(position +0.5)font = FontProperties(fname = r'c:windowsfontssimsun.ttc', size = 10)ax.set_xticklabels(provinceName, fontproperties = font)plt.bar(position, provinceFreq)plt.show()