Scrapy框架: 通用爬虫之SitemapSpider

scrapy startproject cnblogs

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class CnblogsItem(scrapy.Item):
    # define the fields for your item here like:
    # 文章标题
    title = scrapy.Field()
    # 文章url
    url = scrapy.Field()
    # 文章作者
    author = scrapy.Field()

from scrapy.spiders import SitemapSpider
from cnblogs.items import CnblogsItem

class MySpider(SitemapSpider):
    name = 'articles'
    # Sitemap 地址
    sitemap_urls = ['http://www.cnblogs.com/sitemap.xml']
    # 从Sitemap中提取url的规则，并指定回调方法
    sitemap_rules = [
        # 抓取 ***/cate/python/**的url，调用parse_python处理
        ('/cate/python/','parse_python')
    ]

    # 回调方法
    def parse_python(self,response):
        articles = response.css('.post_item')

        for article in articles:
            item = CnblogsItem()
            # 文章标题
            item['title'] = article.css('.titlelnk::text').extract_first()
            # 文章url
            item['url'] = article.css('.titlelnk::attr(href)').extract_first()
            # 文章作者
            item['author'] = article.css('.lightblue::text').extract_first()
            yield item

scrapy crawl articles