我编写了这个爬虫来从Google Play上抓取应用程序评论。我在部分方面取得了成功。我只能提取名称、日期和评论。
我的问题：
我的问题：
- 如何获取所有的评论，因为我只得到了41条。
- 如何从
<div>中获取评级？
import scrapy
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from urlparse import urljoin
class CompItem(scrapy.Item):
rating = scrapy.Field()
data = scrapy.Field()
name = scrapy.Field()
date = scrapy.Field()
class criticspider(CrawlSpider):
name = "gaana"
allowed_domains = ["play.google.com"]
start_urls = ["https://play.google.com/store/apps/details?id=com.gaana&hl=en"]
# rules = (
# Rule(
# SgmlLinkExtractor(allow=('search=jabong&page=1/+',)),
# callback="parse_start_url",
# follow=True),
# )
def parse(self, response):
sites = response.xpath('//div[@class="single-review"]')
items = []
for site in sites:
item = CompItem()
item['data'] = site.xpath('.//div[@class="review-body"]/text()').extract()
item['name'] = site.xpath('.//div/div/span[@class="author-name"]/a/text()').extract()[0]
item['date'] = site.xpath('.//span[@class="review-date"]/text()').extract()[0]
item['rating'] = site.xpath('div[@class="review-info-star-rating"]/aria-label/text()').extract()
items.append(item)
return items