scrapy setting.py 配置

 # Scrapy settings for proxy project 2 # 3 # For simplicity, this file contains only the most important settings by 4 # default. All the other settings are documented here: 5 # 6 #     http://doc.scrapy.org/topics/settings.html 7 # 8  9 BOT_NAME = 'proxy'10 BOT_VERSION = '1.0'11 12 SPIDER_MODULES = ['proxy.spiders']13 NEWSPIDER_MODULE = 'proxy.spiders'14 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)15 16 DOWNLOAD_DELAY = 017 DOWNLOAD_TIMEOUT = 3018 19 ITEM_PIPELINES = [20     'proxy.pipelines.ProxyPipeline'21 ]22 CONCURRENT_ITEMS = 10023 CONCURRENT_REQUESTS_PER_SPIDER = 6424 CONCURRENT_SPIDERS = 12825 26 LOG_ENABLED = True27 LOG_ENCODING = 'utf-8'28 LOG_FILE = '/home/xxx/services_runenv/crawlers/proxy/proxy/log/proxy.log'29 LOG_LEVEL = 'DEBUG'30 LOG_STDOUT = False