from scrapy import Spiderfrom scrapy.http import Requestclass CourseSpider(Spider): name = 'course' allowed_domains = ['coursera.org'] start_urls = ['https://coursera.org/about/partners'] def parse(self, response): listings = response.xpath('//div[@class="rc-PartnerBox vertical-box"]') for listing in listings: title = listing.xpath('.//div[@class="partner-box-wrapper card-one-clicker flex-1"]/p').extract_first() relative_url = listing.xpath('.//a/@href').extract_first() absolute_url = response.urljoin(relative_url) yield Request(response.urljoin(relative_url), callback = self.parse_listing,meta={'title':title,'absolute_url':absolute_url}) def parse_listing(self,response): titles = response.meta.get('title') absolute_url = response.meta.get('absolute_url') titles_course = response.xpath('//div[@class="name headline-1-text"]/text()').extract() url_link = response.xpath('//div[@class="rc-Course"]/a/@href').extract() abs_url = response.urljoin(url_link) yield {'title':title, 'titles':title, 'absolute_url':absolute_url, 'titles_course':titles_course, 'abs_url':abs_url}但是,在通过 cmd 运行脚本时。我收到错误。这些错误表明我不能混合 str 和非 str 参数,我对如何处理这个问题感到困惑。任何帮助,将不胜感激。我尝试添加 extract() 函数,因为它在列表容器上的一些先前的 stackoverflow 问题中被提及以消除该错误,但是我的 xpath 没有获得所需的输出。
添加回答
举报
0/150
提交
取消