class spidermain(object): def __init__(self): self.urls=manager() self.downloader=htmldownloader() self.parser=htmlparser() self.outputer=htmloutputer() def craw(self,root_url): count=1 self.urls.add_new_url(root_url) while self.urls.has_new_url(): try: new_url=self.urls.get_new_url() html_cont=self.downloader.htmldownloader(new_url) print ('carw %d : %s'%(count,new_url)) new_urls,new_data=self.parser.parse(new_url,html_cont) self.urls.add_new_urls(new_urls) self.outputer.collect_data(new_data) if count==1000: break count=count+1 except: print ('craw failed') self.outputer.output_html() if __name__=='__main__': root_url='http://baike.baidu.com/link?url=HiLgunmevYfwmNeyDfoDsrKn4d2OSBg-wcbpXCMCLKo53TPcrL8mGOW-xBdn5Q_gOaMVAQFmXIqkpKNAjQyR2a' obj_spider=spidermain() obj_spider.craw(root_url)#html管理器 html_manager模块class manager(object): def __init__(self): self.new_urls=set() self.old_urls=set() def add_new_url(self,url): if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self,urls): if urls is None or len(urls)==0: return if url in urls: self.add_new_url(url) def has_new_url(self): return len(self.new_urls)!=0 def get_new_url(self): new_url=self.new_urls.pop() self.old_urls.add(new_url) return new_url #html下载器 htmldownloader模块import urllib.requestimport urllib.errorclass htmldownloader(object): def download(self,url): if url is None: return None response=urllib.request.ulropen(url) if response.getcode()!=200: return None return response.read() #html解析器 htmlparser模块from bs4 import BeautifulSoupimport re import urllib.parseclass htmlparser(object): def _get_new_urls(self,page_url,soup): new_urls=set() #<a target="_blank" href="/view/592974.htm">解释器</a> links=soup.find_all('a',href=re.complie(r'/view/\d+\.htm')) for link in links: new_url=link['href'] new_full_url=urllib.parse.urljoin(page_url,new_url) new_urls.add(new_full_url) return new_urls def _get_new_data(self,page_url,soup): res_data={} #url res_data['url']=page_url #<dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1> title_node=soup.find('dd',class_='lemmaWgt-lemmaTitle-title').find('h1') res_data['title']=title_node.get_text() #<div class="lemma-summary" label-module="lemmaSummary"></div> summary_node=soup.find('div',class_='lemma-summary') res_data['summary']=summary_node.get_text() return res_data def parse(self,page_url,html_count): if page_url is None or html_count is None: return soup=BeautifulSoup(html_count,'html.parser',form_encoding='utf-8') new_urls=self._get_new_urls(page_url,soup) new_data=self._get_new_data(page_url,soup) return new_urls,new_data #html输出器 htmloutputer模块class htmloutputer(object): def __init__(self): self.datas=[] def collect_data(self,data): if data is None: return self.datas.append(data) def output_html(self): fout=open('output.html','w') fout.write('<html>') fout.write('<body>') fout.write('<table>') #ascii for data in self.datas: fout.write('<tr>') fout.write('<td>%s</td>'%data['url']) fout.write('<td>%s</td>'%data['title'],edcoding='utf-8') fout.write('<td>%s</td>'%data['summary'],edcoding='utf-8') fout.write('<tr>') fout.write('</table>') fout.write('</body>') fout.write('</html>') fout.close()
添加回答
举报
0/150
提交
取消