Python之Scrapy爬虫框架安装及简单使用详解

pip install Scrapy

pip --version

pip install Scrapy

H:pythonscrapyDemo>scrapy startproject tutorial
New Scrapy project 'tutorial', using template directory 'f:\python27\lib\site-packages\scrapy\templates\project', created in:
  H:pythonscrapyDemotutorial

You can start your first spider with:
  cd tutorial
  scrapy genspider example example.com

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
from scrapy.item import Item, Field
class TutorialItem(Item):
  title = Field()
  author = Field()
  releasedate = Field()

# -*-coding:utf-8-*-
import sys
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from tutorial.items import TutorialItem
reload(sys)
sys.setdefaultencoding("utf-8")
class ListSpider(CrawlSpider):
  # 爬虫名称
  name = "tutorial"
  # 设置下载延时
  download_delay = 1
  # 允许域名
  allowed_domains = ["news.cnblogs.com"]
  # 开始URL
  start_urls = [
    "https://news.cnblogs.com"
  ]
  # 爬取规则,不带callback表示向该类url递归爬取
  rules = (
    Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/page/d',))),
    Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/d+',)), callback='parse_content'),
  )

  # 解析内容函数
  def parse_content(self, response):
    item = TutorialItem()

    # 当前URL
    title = response.selector.xpath('//div[@id="news_title"]')[0].extract().decode('utf-8')
    item['title'] = title

    author = response.selector.xpath('//div[@id="news_info"]/span/a/text()')[0].extract().decode('utf-8')
    item['author'] = author

    releasedate = response.selector.xpath('//div[@id="news_info"]/span[@class="time"]/text()')[0].extract().decode(
      'utf-8')
    item['releasedate'] = releasedate

    yield item

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import codecs
class TutorialPipeline(object):
  def __init__(self):
    self.file = codecs.open('data.json', mode='wb', encoding='utf-8')#数据存储到data.json

  def process_item(self, item, spider):
    line = json.dumps(dict(item)) + "n"
    self.file.write(line.decode("unicode_escape"))

    return item

# -*- coding: utf-8 -*-
BOT_NAME = 'tutorial'
SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'

# 禁止cookies,防止被ban
COOKIES_ENABLED = False
COOKIES_ENABLES = False

# 设置Pipeline,此处实现数据写入文件
ITEM_PIPELINES = {
  'tutorial.pipelines.TutorialPipeline': 300
}

# 设置爬虫爬取的最大深度
DEPTH_LIMIT = 100

from scrapy import cmdline
cmdline.execute("scrapy crawl tutorial".split())