Scrapy日志与调试教程：Scrapy Shell选择器测试技巧

# Spider中记录日志
self.logger.info('信息日志')
self.logger.warning('警告日志')
self.logger.error('错误日志')
self.logger.debug('调试日志')

# Scrapy Shell - 交互式调试
scrapy shell 'https://example.com'

import scrapy

class LoggingSpider(scrapy.Spider):
    name = 'logging_demo'
    start_urls = ['https://quotes.toscrape.com/']

    custom_settings = {
        'LOG_LEVEL': 'INFO',
        'LOG_FILE': 'spider.log',
    }

    def parse(self, response):
        self.logger.info(f'开始解析: {response.url}')

        quotes = response.css('div.quote')
        self.logger.info(f'找到 {len(quotes)} 条名言')

        for i, quote in enumerate(quotes):
            text = quote.css('span.text::text').get()
            if not text:
                self.logger.warning(f'第{i}条名言文本为空')
                continue

            self.logger.debug(f'提取: {text[:30]}...')
            yield {
                'text': text,
                'author': quote.css('small.author::text').get(),
            }

        self.logger.info(f'页面解析完成')

# 启动Shell
scrapy shell 'https://quotes.toscrape.com/'

# 在Shell中测试选择器
>>> response.css('title::text').get()
'Quotes to Scrape'

>>> response.css('div.quote')
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote">\n    <span clas'>, ...]

>>> response.css('span.text::text').get()
'"The world as we have created it is a process of o...'

>>> response.css('span.text::text').getall()
['"The world as we have created it..."', ...]

>>> response.xpath('//span[@class="text"]/text()').get()
'"The world as we have created it..."'

# 在浏览器中查看
>>> view(response)

# 获取新页面
>>> fetch('https://quotes.toscrape.com/page/2/')

# 退出
>>> exit()

import scrapy
from scrapy import signals

class StatsSpider(scrapy.Spider):
    name = 'stats_demo'
    start_urls = ['https://quotes.toscrape.com/']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.item_count = 0
        self.page_count = 0
        self.error_count = 0

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super().from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response):
        self.page_count += 1
        self.logger.info(f'爬取第{self.page_count}页: {response.url}')

        for quote in response.css('div.quote'):
            self.item_count += 1
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
            }

        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

    def spider_closed(self, spider, reason):
        self.logger.info(f'爬虫结束 - 原因: {reason}')
        self.logger.info(f'统计: 页面={self.page_count}, 数据={self.item_count}, 错误={self.error_count}')