调用urlopen时要加timeout, 否则网速不好的时候容易卡死.

html_downloader.py:

# coding: UTF-8
import urllib2

class HtmlDownloader(object):
    def __init__(self):
        pass
        
    def download(self, url):
        if url is None:
            return None
        response = urllib2.urlopen(url,timeout=5)
        
        if response.getcode() != 200:
            print '   --> Can not open url:', url
            return None
            
        html_doc = response.read()
        # print html_doc
               
        return html_doc