Python 爬虫 2 爬取多页网页

import requests
import re

# 将百度贴吧 python吧 的首页源代码获取下来
html = requests.get('http://tieba.baidu.com/f?ie=utf-8&kw=python')
# print html.text

# 这里并没有遇到取不到的情况，所以没有用到hea
# 这个程序没有获得源代码，因为一个网站会对访问他的程序进行检查
# hea是我们自己构造的一个字典，里面保存了user-agent
# hea = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36'}
# html = requests.get('http://jp.tingroom.com/yuedu/yd300p/',headers = hea)

html = requests.get('http://jp.tingroom.com/yuedu/yd300p/')
html.encoding = 'utf-8'          #这一行是将编码转为utf-8否则中文会显示乱码。
# print html.text

title = re.findall('color:#666666;">(.*?)</span>',html.text,re.S)
for each in title:
    print each

chinese = re.findall('color: #039;">(.*?)</a>',html.text,re.S)
for each in chinese:
    print each

url = 'https://www.crowdfunder.com/browse/deals'
html = requests.get(url).text
# print html

# 因为这个网站的例子，已经不用 show more，而改成页码了，所以方法和上次练习的方法一样，下面这个代码没有用了，它只能搜到第一个页面的内容
# 构造字典 data
# 注意这里的page后面跟的数字需要放到引号里面。
url = 'https://www.crowdfunder.com/browse/deals&template=false'

data = {
    'entities_only':'true',
    'page':'2'
}
html_post = requests.post(url,data=data)
title = re.findall('"card-title">(.*?)</div>',html_post.text,re.S)
for each in title:
    print each

# 用这个网址来取标题
url = 'https://www.crowdfunder.com/?q=filter&page=3'
html = requests.get(url).text
# html = requests.get(url).text
title = re.findall('"card-title">(.*?)</div>', html, re.S)
for each in title:
    print each

url = 'https://www.crowdfunder.com/?q=filter&page=1'
for i in range(1,4):
    print i
    
    new_link = re.sub('page=d+','page=%d'%i, url, re.S)
    print new_link
    
    title = re.findall('"card-title">(.*?)</div>', new_link, re.S)
    print title                            # title是空的，因为规律不对了，需要自己重新找一下规律
    
    for each in title:
        print each

url = 'http://www.jikexueyuan.com/course/?pageNum=2'
html = requests.get(url).text
# print html

classinfo = re.findall('(<li id=.*?</li>)', html, re.S)
# for each in classinfo:
#     print each

class spider(object):
    def __init__(self):
        print u'开始爬取内容。。。'

#changepage用来生产不同页数的链接
    def changepage(self,url,total_page):
        now_page = int(re.search('pageNum=(d+)',url,re.S).group(1))
        page_group = []
        for i in range(now_page,total_page+1):
            link = re.sub('pageNum=d+','pageNum=%s'%i,url,re.S)
            page_group.append(link)
        return page_group
    
#getsource用来获取网页源代码
    def getsource(self,url):
        html = requests.get(url)
        return html.text
    
#geteveryclass用来抓取每个课程块的信息
    def geteveryclass(self,source):
        #everyclass = re.findall('(<li deg="".*?</li>)',source,re.S)                      # This code is old.
        everyclass = re.findall('(<li id=.*?</li>)',source,re.S)
        return everyclass
    
#getinfo用来从每个课程块中提取出我们需要的信息
    def getinfo(self,eachclass):
        info = {}
        #info['title'] = re.search('target="_blank">(.*?)</a>',eachclass,re.S).group(1)   # This code is old.
        info['title'] = re.search('title="(.*?)"',eachclass,re.S).group(1)
        
        #info['content'] = re.search('</h2><p>(.*?)</p>',eachclass,re.S).group(1)         # This code is old.
        info['content'] = re.search('<p style="height: 0px; opacity: 0; display: none;">(.*?)</p>',eachclass,re.S).group(1)
        
        timeandlevel = re.findall('<em>(.*?)</em>',eachclass,re.S)
        info['classtime'] = timeandlevel[0]
        info['classlevel'] = timeandlevel[1]
        info['learnnum'] = re.search('"learn-number">(.*?)</em>',eachclass,re.S).group(1)
        return info

#saveinfo用来保存结果到info.txt文件中
    def saveinfo(self,classinfo):
        f = open('info3.txt','a')
        for each in classinfo:
            f.writelines('title:' + each['title'] + 'n')
            f.writelines('content:' + each['content'] + 'n')
            f.writelines('classtime:' + each['classtime'] + 'n')
            f.writelines('classlevel:' + each['classlevel'] + 'n')
            f.writelines('learnnum:' + each['learnnum'] +'nn')
        f.close()

if __name__ == '__main__':

    classinfo = []                                          #存放最终结果
    url = 'http://www.jikexueyuan.com/course/?pageNum=1'    
    jikespider = spider()
    all_links = jikespider.changepage(url,5)                #产生不同的页码的链接
    
    for link in all_links:
        print u'正在处理页面：' + link
        html = jikespider.getsource(link)                   #每个链接，先获取网页源代码
        everyclass = jikespider.geteveryclass(html)         #每个页面内，先抓每个课程的版块
        for each in everyclass:
            info = jikespider.getinfo(each)                 #每个课程版块内，抓title，content，time，level，people，存到字典里
            classinfo.append(info)

    jikespider.saveinfo(classinfo)                          #最终结果保存到txt文件中

print 'done'