# coding:utf-8

# 载入url管理器，解析器，下载器，输出器模块

from test import html_parser, url_manger, html_download, html_outputer

# 定义爬虫类

class SpriderMain(object):

    # 初始化方法

    def _init_(self):

        self.urls = url_manger.urlManager()

        self.downloader = html_download.HtmlDownload()

        self.parser = html_parser.HtmlParser()

        self.outputer = html_outputer.HtmlOutputer()

    def craw(self, root_url):

        count = 1

        self.urls.add_new_url(root_url)

        while self.urls.has_new_url():

            try:

                new_url = self.urls.get_new_url()

                print 'craw %d : %s' % (count, new_url)

                html_cont = self.downloader.download(new_url)

                new_urls, new_data = self.parser.parse(new_url, html_cont)

                self.urls.add_new_urls(new_urls)

                self.outputer.collect_data(new_data)

                if count == 1000:

                    break

                count = count + 1

            except:

                print 'craw failed'

        self.outputer.output_html() 

if __name__ == "__main__":

    root_url = "https://baike.baidu.com/item/%E5%85%A8%E7%94%BB%E5%B9%85%E7%9B%B8%E6%9C%BA?fromtitle=%E5%85%A8%E5%B9%85%E6%9C%BA&fromid=16852355"

    obj_spipder = SpriderMain()

    obj_spipder.craw(root_url)