scrapy爬虫自动爬取的实例

import scrapy
from myproject.items import MyItem

class MySpider(scrapy.Spider):
    '''
    一个回调函数中返回多个Request对象和Item
    '''
    name = 'example.com'
    allowed_domains = ['example.com']
    start_urls = [
        'http://www.example.com/1.html',
        'http://www.example.com/2.html',
        'http://www.example.com/3.html',
    ]

    def parse(self, response):
        sel = scrapy.Selector(response)
        for h3 in response.xpath('//h3').extract():
            yield MyItem(title=h3)         

        for url in response.xpath('//a/@href').extract():
            yield scrapy.Request(url, callback=self.parse)

import scrapy
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
 
class MySpider(CrawlSpider):
    name = 'example.com'
    allowed_domains = ['example.com']
    start_urls = ['http://www.example.com']
 
    # 在如下的规则中，第一条表示匹配category.php但是不匹配subsection.php（没有callback意味着follow为True表示跟进链接)
   
    # 在如下的规则中，第二条表示表示匹配item.php，并使用spider的parse_item方法进行分析。
   
    rules = (
        Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
        Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
    )
 
    def parse_item(self, response):
        self.log('Hi, this is an item page! %s' % response.url)
 
        item = scrapy.Item()
        item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
        item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
        item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
        return item