我正在尝试通过脚本编程调用一个爬虫。我无法通过CrawlerProcess的构造函数重写设置。让我举个例子,使用官方scrapy网站上抓取引号的默认爬虫(最后一个代码片段位于official scrapy quotes example spider)。
class QuotesSpider(Spider):
name = "quotes"
def __init__(self, somestring, *args, **kwargs):
super(QuotesSpider, self).__init__(*args, **kwargs)
self.somestring = somestring
self.custom_settings = kwargs
def start_requests(self):
urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
]
for url in urls:
yield Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.css('small.author::text').extract_first(),
'tags': quote.css('div.tags a.tag::text').extract(),
}
这是我尝试运行报价爬虫的脚本。
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.settings import Settings
def main():
proc = CrawlerProcess(get_project_settings())
custom_settings_spider = \
{
'FEED_URI': 'quotes.csv',
'LOG_FILE': 'quotes.log'
}
proc.crawl('quotes', 'dummyinput', **custom_settings_spider)
proc.start()
CrawlerProcess
并更新设置时,您可以使用for
循环,而不是覆盖custom_settings
。 - eLRuLLCrawlerRunner
按顺序运行爬虫? - paul trmbrth