Scrapy分页爬取详解 - 从简单分页到多级分页完整指南

def parse(self, response):
    # 提取当前页数据
    for item in response.css('div.item'):
        yield {
            'name': item.css('h3::text').get(),
            'price': item.css('.price::text').get(),
        }

    # 查找下一页链接
    next_page = response.css('a.next::attr(href)').get()
    if next_page:
        # 使用urljoin将相对URL转为绝对URL
        yield scrapy.Request(
            url=response.urljoin(next_page),
            callback=self.parse,  # 回调自身实现循环
        )

import scrapy

class SimplePaginationSpider(scrapy.Spider):
    name = 'simple_pagination'
    start_urls = ['https://quotes.toscrape.com/']

    def parse(self, response):
        # 提取当前页的名言数据
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
                'tags': quote.css('div.tags a.tag::text').getall(),
            }

        # 查找并跟进下一页
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

import scrapy

class UrlPatternSpider(scrapy.Spider):
    name = 'url_pattern'
    allowed_domains = ['example.com']

    def start_requests(self):
        base_url = 'https://example.com/products'
        # 生成第1-10页的URL并批量请求
        for page in range(1, 11):
            url = f'{base_url}?page={page}'
            yield scrapy.Request(
                url,
                callback=self.parse,
                meta={'page': page},  # 传递页码信息
            )

    def parse(self, response):
        page = response.meta['page']
        self.logger.info(f'正在爬取第{page}页')

        for product in response.css('div.product'):
            yield {
                'name': product.css('h3::text').get(),
                'price': product.css('.price::text').get(),
                'page': page,
            }

import scrapy

class MultiLevelPaginationSpider(scrapy.Spider):
    name = 'multi_pagination'
    start_urls = ['https://quotes.toscrape.com/']

    def parse(self, response):
        """列表页：提取数据并跟进详情页和下一页"""
        for quote in response.css('div.quote'):
            # 提取基本信息到Item
            item = {
                'text': quote.css('span.text::text').get(),
                'author_name': quote.css('small.author::text').get(),
            }

            # 跟进作者详情页获取更多信息
            author_url = quote.css('small.author + a::attr(href)').get()
            if author_url:
                yield response.follow(
                    author_url,
                    callback=self.parse_author,
                    meta={'item': item},  # 传递基础数据
                )
            else:
                yield item

        # 跟进列表的下一页
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

    def parse_author(self, response):
        """作者详情页：补充作者详细信息"""
        item = response.meta['item']  # 获取传递过来的基础数据
        item['author_born_date'] = response.css(
            'span.author-born-date::text'
        ).get()
        item['author_born_location'] = response.css(
            'span.author-born-location::text'
        ).get()
        item['author_description'] = response.css(
            'span.author-description::text'
        ).get()
        yield item

import scrapy

class LimitedPaginationSpider(scrapy.Spider):
    name = 'limited_pagination'
    start_urls = ['https://quotes.toscrape.com/']
    max_pages = 5       # 最大爬取页数
    items_count = 0     # 已采集数量计数器

    def parse(self, response):
        for quote in response.css('div.quote'):
            self.items_count += 1
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
            }

        # 分页终止条件：达到最大页数或采集数量上限
        current_page = response.meta.get('page', 1)
        if current_page < self.max_pages:
            next_page = response.css('li.next a::attr(href)').get()
            if next_page:
                yield response.follow(
                    next_page,
                    callback=self.parse,
                    meta={'page': current_page + 1},
                )