我的代码：spide_main页面from baike_spider import url_manager, html_downloader, html_parser,\ html_outputerclass SpiderMain(object): def __init__(self): self.urls=url_manager.UrlManager() self.downloader=html_downloader.HtmlDownloader() self.parser=html_parser.HtmlParser() self.outputer=html_outputer.HtmlOutputer() def craw(self, root_url): count=1 self.urls.add_new_url(root_url) while self.urls.has_new_url(): try: new_url=self.urls.get_new_url() print 'craw %d : %s' % (count,new_url) html_cont=self.downloader.download(new_url) new_urls,new_data=self.parser.parse(new_url,html_cont) self.urls.add_new_urls(new_urls) self.outputer.collect_data(new_data) if count==1000: break count=count+1 except: print 'craw failed' self.outputer.output_html()if __name__=="_main_": root_url="http://baike.baidu.com/view/21087.htm" obj_spider=SpiderMain() obj_spider.craw(root_url)url_manager页面class UrlManager(object): def __init__(self): self.new_urls = set() self.old_urls = set() def add_new_url(self,url): if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self,urls): if urls is None or len(urls) == 0: return for url in urls: self.add_new_url(url) def has_new_url(self): return len(self.new_urls) != 0 def get_new_url(self): new_url=self.new_urls.pop() self.old_urls.add(new_url) return new_urlhtml_downloader页面import urllib2class HtmlDownloader(object): def download(self,url): if url is None: return None response = urllib2.urlopen(url) if response.getcode() != 200: return None return response.read() html_parser页面import urllib2class HtmlDownloader(object): def download(self,url): if url is None: return None response = urllib2.urlopen(url) if response.getcode() != 200: return None return response.read()html_outputer页面class HtmlOutputer(object): def __init__(self): self.datas = [] def collect_data(self,data): if data is None: return self.datas.append(data) def output_html(self): fout=open('output.html','w') fout.write("<html>") fout.write("<body>") fout.write("<table>") #ascii for data in self.datas: fout.write("<tr>") fout.write("<td>%s</td>" % data['url']) fout.write("<td>%s</td>" % data['title'].encode('utf-8')) fout.write("<td>%s</td>" % data['summary'].encode('utf-8')) fout.write("</tr>") fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close() 运行后没有反应，
添加回答
举报
0/150
提交
取消