我有一个可以抓取单个页面的功能。跟随相应的链接后,如何抓取多个页面?我是否需要像下面的gotoIndivPage()一样调用parse()的单独函数?谢谢!
import scrapy
class trainingScraper(scrapy,Spider):
name = "..."
start_urls = "url with links to multiple pages"
# for scraping individual page
def parse(self,response):
SELECTOR1 = '.entry-title ::text'
SELECTOR2 = '//li[@class="location"]/ul/li/a/text()'
yield{
'title': response.css(SELECTOR1).extract_first(),'date': response.xpath(SELECTOR2).extract_first(),}
def gotoIndivPage(self,response):
PAGE_SELECTOR = '//h3[@class="entry-title"]/a/@href'
for page in response.xpath(PAGE_SELECTOR):
if page:
yield scrapy.Request(
response.urljoin(page),callback=self.parse
)