python小应用-博客园隐藏的小秘密及各类排行榜【附源码】

import requests
import re
import json
import time

CRAWLING_URL = 'https://www.cnblogs.com/'

def downloadPage():
    """获取页面内容"""
    print('获取页面内容用时')
    url = CRAWLING_URL

    res = requests.get(url).text
    html = BeautifulSoup(res, 'lxml')

    data = {}
    postList = html.find_all(class_='post_item_foot')
    for postInfo in postList:
        content = postInfo.contents
        # 发布时间字符串
        timeStr = content[2][11:27]

        localTime = time.localtime(time.mktime(
            time.strptime(timeStr, '%Y-%m-%d %H:%M')))
        # 以2018-11-01 15 格式时间 作为 key
        timeIndex = time.strftime("%Y-%m-%d %H", localTime)
        viewStr = content[4].contents[0].contents[0]
        commontStr = content[3].contents[0].contents[0]

        # 浏览量
        view = int(re.findall("d+", viewStr)[0])
        # 评论量
        commont = int(re.findall("d+", commontStr)[0])

        if timeIndex in data:
            data[timeIndex]['view'] += view
            data[timeIndex]['commont'] += commont
            data[timeIndex]['postCount'] += 1
        else:
            data[timeIndex] = {
                'view': view,
                'commont': commont,
                'postCount': 1
            }
    
    return data

# ...
CRAWLING_URL = 'https://www.cnblogs.com/#p'
def crawlData(page, data):
    url = CRAWLING_URL + str(page)
    # ...
 
def main():
    pageNum = 11
    data = {}
    for page in range(1, pageNum):
        data = crawlData(page, data)
        page += 1
    print(data)

# ...
CRAWLING_URL = 'https://www.cnblogs.com/mvc/AggSite/PostList.aspx'

def crawlData(page, data):
    """获取页面内容"""
    url = CRAWLING_URL
    headers = {
        'Content-Type': 'application/json',
    }
    params = json.dumps({
        'CategoryId': 808,
        'CategoryType': "SiteHome",
        'ItemListActionName': "PostList",
        'PageIndex': page,
        'ParentCategoryId': 0,
        'TotalPostCount': 4000,
    })
    res = requests.post(url, data=params, headers=headers, verify=False).text
    # ...

def main():
    pageNum = 11
    data = {}
    # 获取所有数据
    for page in range(1, pageNum):
        data = crawlData(page, data)
        print('已完成: %s/%s' % (page, pageNum - 1))
        page += 1

    # excel 表格存储
    col = 2
    ws1['A1'] = '日期'
    ws1['B1'] = '查看人数'
    ws1['C1'] = '评论人数'
    ws1['D1'] = '发布数量'
    for postCount in data:
        col_A = 'A%s' % col
        col_B = 'B%s' % col
        col_C = 'C%s' % col
        col_D = 'D%s' % col
        col_E = 'E%s' % col
        col_F = 'F%s' % col
        ws1[col_A] = postCount
        ws1[col_B] = data[postCount]['view']
        ws1[col_C] = data[postCount]['commont']
        ws1[col_D] = data[postCount]['postCount']
        col += 1
            
    wb.save(filename=dest_filename)
    print('-------------SUCCESS--------------')