首页手记爬取猫眼电影

爬取猫眼电影

标签：

Python 爬虫

针对构建的电影购票功能获取相应数据，主要获取：电影信息+影院信息+场次信息，都不难获取，主要是场次信息方面的构造、排布

1. 爬取首页所有电影链接

    def parse(self, response):
        movie_list = response.xpath('//dd//div[@class="movie-item"]/a/@href').extract()
        for url in movie_list:
            yield scrapy.Request("https://maoyan.com" + url, callback=self.parse_tag)

2. 进入子页面获取电影具体信息

    def parse_tag(self, response):
        item = MaoyanItem()
        item['movie_name'] = response.xpath('//h3[@class="name"]/text()').extract_first()
        item['ellipsis'] = response.xpath('//div[@class="ename ellipsis"]/text()').extract()
        time_str = ''.join(response.xpath('//li[@class="ellipsis"]//text()').extract())
        item['time'] = self.date_reg_exp.findall(time_str)[0]
        if response.xpath('//i[@class="imax3d"]'):
            item['vision'] = "3DIMAX"
        elif response.xpath('//i[@class="m3d"]'):
            item['vision'] = "3D"
        else:
            item['vision'] = ""
        yield item

3. 获取影院信息

因为这里需要的数据并不多，就自行选取部分区域爬取即可，想要爬取所有区域或者城市可以根据具体的城市id以及区域唯一标识id进行构造

4. 获取影院详情信息

        def parse_tag(self, response):
        item = MaoyanItem()
        item['movie_name'] = response.xpath('//h3[@class="name"]/text()').extract_first()
        item['ellipsis'] = response.xpath('//div[@class="ename ellipsis"]/text()').extract()
        time_str = ''.join(response.xpath('//li[@class="ellipsis"]//text()').extract())
        item['time'] = self.date_reg_exp.findall(time_str)[0]
        item['introduction'] = response.xpath('normalize-space(//span[@class="dra"]/text())').extract_first()
        duration = ''.join(response.xpath('//li[@class="ellipsis"][2]//text()').extract())
        item['duration'] = re.findall('\d+', duration)[0]
        if response.xpath('//i[@class="imax3d"]'):
            item['vision'] = "3DIMAX"
        elif response.xpath('//i[@class="m3d"]'):
            item['vision'] = "3D"
        else:
            item['vision'] = ""
        yield item

5. 获取场次信息

场次信息我用的json文件来保存,因为其中涉及到不同日期不同场次，针对不同电影，信息也是不同的，这是一个比较庞大的数据，我们按照如下形式保存

'''
    movie_session = [不同电影的场次数据
        {movie_name: ...
         result:[每一天的场次数据
            {
                date: ...
                session_result:[当天的场次数据
                    {},
                    {}....
                ]        
            }
         ]
        }
    ]：每部电影的数据
'''

去对应影院的详情页面，对电影、日期进行循环，然后抓取到所有的场次列表，写入到文件当中

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
    "Referer": "https://maoyan.com/xseats/201906020127664?movieId=246061&cinemaId=17372",
    "Host": "maoyan.com",
    "Upgrade-Insecure-Requests": "1",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
}
movie_result = []

def getCinemas(urls):
    cinemas = []
    for url in urls:
        html = etree.HTML(getHtml(url))
        cinemas.extend(html.xpath('//div[@class="cinema-info"]/a/@href'))
    return cinemas


def getUrls(urls):
    url_list = []
    cinemas = getCinemas(urls)
    for cinema in cinemas:
        cinema_url = "http://maoyan.com" + cinema
        url_list.append(cinema_url)
    return url_list


def getHtml(url):
    return requests.get(url, headers=headers).text


#给出当天的xpath对象获取对应日期的场次数据
def getToday(today, time_list, num):
    result = []
    session_result = []
    session_list = today.xpath('.//table/tbody/tr')
    # 当天每个场次
    for session in session_list:
        begin_time = session.xpath('.//span[@class="begin-time"]/text()')[0]
        end_time = session.xpath('.//span[@class="end-time"]/text()')[0]
        lang = session.xpath('.//span[@class="lang"]/text()')[0]
        hall = session.xpath('.//span[@class="hall"]/text()')[0]
        session_result.append({
            "date": re.findall('\d+', time_list[num])[0] + '-' + re.findall('\d+', time_list[num])[1],
            "begin_time": begin_time,
            "end_time": end_time,
            "lang": lang,
            "hall": hall
        })
    result.append({
        "date": re.findall('\d+', time_list[num])[0] + '-' + re.findall('\d+', time_list[num])[1],
        "session_result": session_result
    })
    return result


def parse_tag(html, days):
    movie_session = []
    movie_name = html.xpath('//h3[@class="movie-name"]/text()')[0]
    time_list = html.xpath('//div[@class="show-list active"]//div[@class="show-date"]/span/text()')[1:]
    for day in days:
        result = getToday(day, time_list)
        movie_session.append({
            "movie_name": movie_name,
            "result": result
        })
    return movie_session

def test(text):
    movies_session = []
    result = []
    tmp_result = []
    html = etree.HTML(text)
    cinema_name = html.xpath('//h3[@class="name text-ellipsis"]/text()')[0]
    movies_list = html.xpath('//div[contains(@class, "show-list")]')

    #每一部电影数据
    for movie in movies_list:
        tmp_result.append(get_MovieData(movie)[0])
    with open('files/' + cinema_name + '.json', 'a+', encoding='utf-8') as f:
        f.write(json.dumps(tmp_result, ensure_ascii=False, indent=2))


def get_MovieData(movie):
    movie_session = []
    result = []
    movie_name = movie.xpath('.//h3[@class="movie-name"]/text()')[0]
    if movie.xpath('.//span[@class="score sc"]/text()'):
        star = movie.xpath('.//span[@class="score sc"]/text()')[0]
    else:
        star = '暂无评分'

    time_list = movie.xpath('.//div[@class="show-date"]/span/text()')[1:]
    #所有场次天数
    days = movie.xpath('.//div[contains(@class, "plist-container")]')
    for day in days:
        result.append(getToday(day, time_list, days.index(day)))
        # print(movie_result)
    movie_session.append({
        "movie_name": movie_name,
        "star": star,
        "result": result
    })

    return movie_session


def parse(text):
    html = etree.HTML(text)
    movie_session = []
    result = []
    session_result = []

    list = html.xpath('//div[@class="show-list active"]')
    movie_name = html.xpath('//h3[@class="movie-name"]/text()')[0]
    star = html.xpath('//span[@class="score sc"]/text()')[0]
    time_list = html.xpath('//div[@class="show-list active"]//div[@class="show-date"]/span/text()')[1:]#6月2， 6月3
    days = html.xpath('//div[@data-index="0"]//div[contains(@class, "plist-container")]')
    for day in days:
        print(json.dumps(getToday(day, time_list, days.index(day)), ensure_ascii=False))


if __name__ == '__main__':
    urls = [
        "https://maoyan.com/cinemas?districtId=3799",
        "https://maoyan.com/cinemas?districtId=3798",
        "https://maoyan.com/cinemas?areaId=-1&districtId=3802",
    ]
    cinemas = getUrls(urls)
    for url in cinemas:
        test(requests.get(url, headers=headers).text)