Scrapy爬虫配置指南：并发控制/延迟/重试全解析

# settings.py - 全局配置
SETTING_NAME = value

# Spider中覆盖 - Spider级别配置
class MySpider(scrapy.Spider):
    custom_settings = {
        'SETTING_NAME': value,
    }

# 命令行覆盖 - 最高优先级
scrapy crawl spider -s SETTING_NAME=value

# settings.py - 生产环境推荐配置

BOT_NAME = 'myproject'
SPIDER_MODULES = ['myproject.spiders']
NEWSPIDER_MODULE = 'myproject.spiders'

# 遵守robots.txt（根据需求调整）
ROBOTSTXT_OBEY = False

# 并发和延迟
CONCURRENT_REQUESTS = 8
DOWNLOAD_DELAY = 2
CONCURRENT_REQUESTS_PER_DOMAIN = 4

# 请求头设置
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0'

# 重试配置
RETRY_ENABLED = True
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429, 403]

# 超时设置
DOWNLOAD_TIMEOUT = 30

# Cookie设置
COOKIES_ENABLED = True

# 日志设置
LOG_LEVEL = 'INFO'
LOG_FILE = 'scrapy.log'

import scrapy

class AggressiveSpider(scrapy.Spider):
    """激进型Spider - 高并发低延迟"""
    name = 'aggressive'

    # Spider级别覆盖settings
    custom_settings = {
        'CONCURRENT_REQUESTS': 32,
        'DOWNLOAD_DELAY': 0.5,
        'RETRY_TIMES': 5,
        'LOG_LEVEL': 'DEBUG',
    }

    start_urls = ['https://example.com']

    def parse(self, response):
        yield {'title': response.css('title::text').get()}


class GentleSpider(scrapy.Spider):
    """温和型Spider - 低并发高延迟"""
    name = 'gentle'

    custom_settings = {
        'CONCURRENT_REQUESTS': 2,
        'DOWNLOAD_DELAY': 5,
        'AUTOTHROTTLE_ENABLED': True,
    }

    start_urls = ['https://example.com']

    def parse(self, response):
        yield {'title': response.css('title::text').get()}

# settings.py - 自动限速配置

# 启用自动限速
AUTOTHROTTLE_ENABLED = True

# 初始延迟（秒）
AUTOTHROTTLE_START_DELAY = 2

# 最大延迟（秒）
AUTOTHROTTLE_MAX_DELAY = 30

# 目标并发数
AUTOTHROTTLE_TARGET_CONCURRENCY = 4.0

# 显示调试信息（生产环境设为False）
AUTOTHROTTLE_DEBUG = True

# 启用HTTP缓存（开发调试用）
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 86400  # 缓存过期时间：1天
HTTPCACHE_DIR = 'httpcache'