数据分析入门系列教程-微博热点

import requests
import json
from bs4 import BeautifulSoup
import pandas as pd
import timeHeaders = {'Cookie': 'SINAGLOBAL=4979979695709.662.1540896279940; SUB=_2AkMrYbTuf8PxqwJRmPkVyG_nb45wwwHEieKdPUU1JRMxHRl-yT83qnI9tRB6AOGaAcavhZVIZBiCoxtgPDNVspj9jtju; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9W5d4hHnVEbZCn4G2L775Qe1; _s_tentry=-; Apache=1711120851984.973.1564019682028; ULV=1564019682040:7:2:1:1711120851984.973.1564019682028:1563525180101; login_sid_t=8e1b73050dedb94d4996a67f8d74e464; cross_origin_proto=SSL; Ugrow-G0=140ad66ad7317901fc818d7fd7743564; YF-V5-G0=95d69db6bf5dfdb71f82a9b7f3eb261a; WBStorage=edfd723f2928ec64|undefined; UOR=bbs.51testing.com,widget.weibo.com,www.baidu.com; wb_view_log=1366*7681; WBtopGlobal_register_version=307744aa77dd5677; YF-Page-G0=580fe01acc9791e17cca20c5fa377d00|1564363890|1564363890'}def sister(page):
    sister = []
    for i in range(0, page):
        print("page: ", i)
        url = 'https://weibo.com/aj/v6/comment/big?ajwvr=6&id=4380261561116383&page=%s' % int(i)
        req = requests.get(url, headers=Headers).text
        html = json.loads(req)['data']['html']
        content = BeautifulSoup(html, "html.parser")
        comment_text = content.find_all('div', attrs={'class': 'WB_text'})
        for c in comment_text:
            sister_text = c.text.split("：")[1]
            sister.append(sister_text)
        time.sleep(5)    return sister

if __name__ == '__main__':
    print("start")
    sister_comment = sister(1001)
    sister_pd = pd.DataFrame(columns=['sister_comment'], data=sister_comment)
    sister_pd.to_csv('sister.csv', encoding='utf-8')

def get_uid(name):
    try:
        url = 'https://s.weibo.com/user?q=%s' % name
        res = requests.get(url).text
        content = BeautifulSoup(res, 'html.parser')
        user = content.find('div', attrs={'class': 'card card-user-b s-pg16 s-brt1'})
        user_info = user.find('div', attrs={'class': 'info'}).find('div')
        href_list = user_info.find_all('a')
        if len(href_list) == 3:
            title = href_list[1].get('title')
            if title == '微博个人认证':
                uid = href_list[2].get('uid')
                return uid
            elif title == '微博会员':
                uid = href_list[2].get('uid')
                return uid
        else:
            print("There are something wrong")
            return False
    except:
        raise

def get_userinfo(uid):
    try:
        url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=%s' % uid
        res = requests.get(url).json()
        containerid = res['data']['tabsInfo']['tabs'][1]['containerid']
        mblog_counts = res['data']['userInfo']['statuses_count']
        followers_count = res['data']['userInfo']['followers_count']
        userinfo = {
            "containerid": containerid,
            "mblog_counts": mblog_counts,
            "followers_count": followers_count
        }
        return userinfo
    except:
        raise

def get_blog_info(cards, i, name, page):
    blog_dict = {}
    if cards[i]['card_type'] == 9:
        scheme = cards[i]['scheme']  # 微博地址
        mblog = cards[i]['mblog']
        mblog_text = mblog['text']
        create_time = mblog['created_at']
        mblog_id = mblog['id']
        reposts_count = mblog['reposts_count']  # 转发数量
        comments_count = mblog['comments_count']  # 评论数量
        attitudes_count = mblog['attitudes_count']  # 点赞数量
        with open(name, 'a', encoding='utf-8') as f:
            f.write("----第" + str(page) + "页，第" + str(i + 1) + "条微博----" + "n")
            f.write("微博地址：" + str(scheme) + "n" + "发布时间：" + str(create_time) + "n"
                    + "微博内容：" + mblog_text + "n" + "点赞数：" + str(attitudes_count) + "n"
                    + "评论数：" + str(comments_count) + "n" + "转发数：" + str(reposts_count) + "n")
        blog_dict['mblog_id'] = mblog_id
        blog_dict['mblog_text'] = mblog_text
        blog_dict['create_time'] = create_time
        return blog_dict
    else:
        print("没有任何微博哦")
        return False

def get_blog_by_text(containerid, blog_text, name):
    blog_list = []
    page = 1
    while True:
        try:
            url = 'https://m.weibo.cn/api/container/getIndex?containerid=%s&page=%s' % (containerid, page)
            res_code = requests.get(url).status_code
            if res_code == 418:
                print("访问太频繁，过会再试试吧")
                return False
            res = requests.get(url).json()
            cards = res['data']['cards']
            if len(cards) > 0:
                for i in range(len(cards)):
                    print("-----正在爬取第" + str(page) + "页，第" + str(i+1) + "条微博------")
                    blog_dict = get_blog_info(cards, i, name, page)
                    blog_list.append(blog_dict)
                    if blog_list is False:
                        break
                    mblog_text = blog_dict['mblog_text']
                    create_time = blog_dict['create_time']
                    if blog_text in mblog_text:
                        print("找到相关微博")
                        return blog_dict['mblog_id']
                    elif checkTime(create_time, config.day) is False:
                        print("没有找到相关微博")
                        return blog_list
                page += 1
                time.sleep(config.sleep_time)
            else:
                print("没有任何微博哦")
                break        except:
            pass

def checkTime(inputtime, day):
    try:
        intime = datetime.datetime.strptime("2019-" + inputtime, '%Y-%m-%d')
    except:
        return "时间转换失败"    now = datetime.datetime.now()
    n_days = now - intime
    days = n_days.days
    if days < day:
        return True
    else:
        return False

day = 90  # 最久抓取的微博时间，60即为只抓取两个月前到现在的微博
sleep_time = 5  # 延迟时间，建议配置5-10s

def get_comment(self, mblog_id, page):
    comment = []
    for i in range(0, page):
        print("-----正在爬取第" + str(i) + "页评论")
        url = 'https://weibo.com/aj/v6/comment/big?ajwvr=6&id=%s&page=%s' % (mblog_id, i)
        req = requests.get(url, headers=self.headers).text
        html = json.loads(req)['data']['html']
        content = BeautifulSoup(html, "html.parser")
        comment_text = content.find_all('div', attrs={'class': 'WB_text'})
        for c in comment_text:
            _text = c.text.split("：")[1]
            comment.append(_text)
        time.sleep(config.sleep_time)    return commentdef download_comment(self, comment):
    comment_pd = pd.DataFrame(columns=['comment'], data=comment)
    timestamp = str(int(time.time()))
    comment_pd.to_csv(timestamp + 'comment.csv', encoding='utf-8')

from weibo_spider import WeiBo
from config import headers

def main(name, spider_type, text, page, iscomment, comment_page):
    print("开始...")
    weibo = WeiBo(name, headers)
    print("以名字搜索...")
    print("获取 UID...")
    uid = weibo.get_uid()
    print("获取 UID 结束")
    print("获取用户信息...")
    userinfo = weibo.get_userinfo(uid)
    print("获取用户信息结束")
    if spider_type == "Text" or spider_type == "text":
        print("爬取微博...")
        blog_info = weibo.get_blog_by_text(userinfo['containerid'], text, name)
        if isinstance(blog_info, str):
            print("搜索到微博，爬取成功")
            if iscomment == "Yes" or iscomment == "YES" or iscomment == "yes":
                print("现在爬取评论")
                comment_info = weibo.get_comment(blog_info, comment_page)
                weibo.download_comment(comment_info)
                print("评论爬取成功，请查看文件")
                return True
            return True
        else:
            print("没有搜索到微博，爬取结束")
            return False
    elif spider_type == "Page" or spider_type == "page":
        blog_info = weibo.get_blog_by_page(userinfo['containerid'], page, name)
        if blog_info and len(blog_info) > 0:
            print("爬取成功，请查看文件")
            return True
    else:
        print("请输入正确选项")
        return Falseif __name__ == '__main__':
    target_name = input("type the name: ")
    spider_type = input("type spider type(Text or Page): ")
    text = "你好"
    page_count = 10
    iscomment = "No"
    comment_page_count = 100
    while spider_type not in ("Text", "text", "Page", "page"):
        spider_type = input("type spider type(Text or Page): ")
    if spider_type == "Page" or spider_type == "page":
        page_count = input("type page count(Max is 50): ")
        while int(page_count) > 50:
            page_count = input("type page count(Max is 50): ")
    elif spider_type == "Text" or spider_type == "text":
        text = input("type blog text for search: ")
        iscomment = input("type need crawl comment or not(Yes or No): ")
        while iscomment not in ("Yes", "YES", "yes", "No", "NO", "no"):
            iscomment = input("type need crawl comment or not(Yes or No): ")
        if iscomment == "Yes" or iscomment == "YES" or iscomment == "yes":
            comment_page_count = input("type comment page count(Max is 1000): ")
            while int(comment_page_count) > 1000:
                comment_page_count = input("type comment page count(Max is 1000): ")
    result = main(target_name, spider_type, text, int(page_count), iscomment, int(comment_page_count))
    if result:
        print("爬取成功！！")
    else:
        print("爬取失败！！")

weibo = WeiBo(name, headers)

class WeiBo(object):    def __init__(self, name, headers):
        self.name = name
        self.headers = headers

    def get_uid(self):  # 获取用户的 UID
        ...    def get_userinfo(self, uid):  # 获取用户信息，包括 containerid
        ...    def get_blog_by_page(self, containerid, page, name):  # 获取 page 页的微博信息
        ...    def get_blog_by_text(self, containerid, blog_text, name):  # 一个简单的搜索功能，根据输入的内容查找对应的微博
        ...    def get_comment(self, mblog_id, page):  # 与上个函数配合使用，用于获取某个微博的评论
        ...    def download_comment(self, comment):  # 下载评论

    ...在类的初始化函数中，传入需要爬取的大 V 名称和我们准备好的 headers（cookie），然后把上面写好的函数写到该类下，后面该类的实例 weibo 就能够调用这些函数了。

import datetime
from config import daydef checkTime(inputtime, day):
.    ...def get_blog_info(cards, i, name, page):
    ...