#!/usr/bin/env python

# -*- coding: utf-8 -*-

# @Date    : 2016-08-25 15:14:46

# @Author  : Your Name (you@example.org)

# @Link    : http://example.org

# @Version : $Id$

import url_manager

import html_downloader

import html_parser

import html_outputer

class SpiderMain(object):

def __init__(self):

self.urls = url_manager.UrlManager()

self.downloader = html_downloader.HtmlDownloader()

self.parser = html_parser.HtmlParser()

self.outputer = html_outputer.HtmlOutputer()

def craw(self, root_url):

count = 1

self.urls.add_new_url(root_url)

while self.urls.has_new_url():

try:

new_url = self.urls.get_new_url()

print 'craw %d : %s' % (count, new_url)

html_cont = self.downloader.download(new_url)

new_urls, new_data = self.parser.parse(new_url, html_cont)

self.urls.add_new_urls(new_urls)

self.outputer.collect_data(new_data)

if count == 1000:

break

count = count + 1

except: 

print 'craw failed'

self.outputer.outputer_html()

if __name__=="__main__":

root_url = "http://baike.baidu.com/view/21087.htm"

obj_spider = SpiderMain()

obj_spider.craw(root_url)

输出结果：

Traceback (most recent call last):

  File "spider_main.py", line 49, in <module>

    obj_spider.craw(root_url)

  File "spider_main.py", line 24, in craw

    self.urls.add_new_url(root_url)

TypeError: add_new_url() takes exactly 1 argument (2 given)