Scrapy中间件详解 - 下载器与爬虫中间件完整指南

class MyDownloaderMiddleware:
    def process_request(self, request, spider):
        """处理请求 - 在请求发送到下载器之前调用"""
        # 可以修改请求头、设置代理等
        return None  # 返回None继续处理

    def process_response(self, request, response, spider):
        """处理响应 - 在下载器返回响应后调用"""
        # 可以检查响应内容、处理重试等
        return response  # 必须返回response或request

    def process_exception(self, request, exception, spider):
        """处理异常 - 当下载器或process_request抛出异常时调用"""
        # 可以处理超时、连接错误等
        return None  # 返回None继续处理其他异常处理器

DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.RandomUserAgentMiddleware': 400,
    'myproject.middlewares.ProxyMiddleware': 500,
    'myproject.middlewares.CustomRetryMiddleware': 550,
}

SPIDER_MIDDLEWARES = {
    'myproject.middlewares.MySpiderMiddleware': 500,
}

import random

class RandomUserAgentMiddleware:
    """随机User-Agent中间件 - 每次请求使用不同的浏览器标识"""

    def __init__(self):
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edge/120.0.0.0',
        ]

    def process_request(self, request, spider):
        user_agent = random.choice(self.user_agents)
        request.headers['User-Agent'] = user_agent
        spider.logger.debug(f'使用UA: {user_agent[:50]}...')
        return None

class ProxyMiddleware:
    """代理中间件 - 为请求设置HTTP代理"""

    def __init__(self, proxy_url):
        self.proxy_url = proxy_url

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            proxy_url=crawler.settings.get('PROXY_URL', ''),
        )

    def process_request(self, request, spider):
        if self.proxy_url:
            request.meta['proxy'] = self.proxy_url
            spider.logger.debug(f'使用代理: {self.proxy_url}')
        return None

    def process_exception(self, request, exception, spider):
        spider.logger.warning(f'请求异常: {exception}')
        return None

from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.utils.response import response_status_message

class CustomRetryMiddleware(RetryMiddleware):
    """自定义重试中间件 - 针对特定状态码和异常自动重试"""

    RETRY_TIMES = 3
    RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429]

    def process_response(self, request, response, spider):
        # 检查响应状态码是否需要重试
        if response.status in self.RETRY_HTTP_CODES:
            retry_times = request.meta.get('retry_times', 0)
            if retry_times < self.RETRY_TIMES:
                spider.logger.info(
                    f'重试 {retry_times + 1}/{self.RETRY_TIMES}: '
                    f'{request.url} (状态码: {response.status})'
                )
                request.meta['retry_times'] = retry_times + 1
                request.dont_filter = True
                return request
        return response

    def process_exception(self, request, exception, spider):
        retry_times = request.meta.get('retry_times', 0)
        if retry_times < self.RETRY_TIMES:
            spider.logger.info(
                f'异常重试 {retry_times + 1}/{self.RETRY_TIMES}: {request.url}'
            )
            request.meta['retry_times'] = retry_times + 1
            request.dont_filter = True
            return request.copy()
        return None

import random

class CookiePoolMiddleware:
    """Cookie池中间件 - 随机使用Cookie池中的Cookie"""

    def __init__(self, cookie_list):
        self.cookie_list = cookie_list

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            cookie_list=crawler.settings.getlist('COOKIE_POOL', []),
        )

    def process_request(self, request, spider):
        if self.cookie_list:
            cookie = random.choice(self.cookie_list)
            request.cookies = cookie
            spider.logger.debug(f'使用Cookie: {cookie.get("session_id", "")[:10]}...')
        return None