Python爬虫实例：用requests重构豆瓣热播电影爬虫

功能：

• 用requests重新实现豆瓣热播电影（原先用的是urllib,urlib2）
• 增加功能：下载每一个电影的海报图片

分析海报图片在HTML代码中的格式

<li id="2131940" class="list-item" data-title="魔兽" data-score="8.2" data-star="40" data-release="2016" data-duration="124分钟" data-region="美国 中国大陆 加拿大" data-director="邓肯·琼斯" data-actors="崔维斯·费米尔 / 托比·凯贝尔 / 宝拉·巴顿" data-category="nowplaying" data-enough="True" data-showed="True" data-votecount="63034" data-subject="2131940">    <ul class="">        <li class="poster">            <a href="https://movie.douban.com/subject/2131940/?from=playing_poster" class="ticket-btn" target="_blank" data-psource="poster">                <img src="https://img1.doubanio.com/view/movie_poster_cover/mpst/public/p2345947329.jpg" alt="魔兽" rel="nofollow" class="">            </a>        </li>    </ul>

每个电影的海报图片url在爬取的其电影用的

标签，的内部。

实例

# -*- coding: utf-8 -*-import urllib2import jsonfrom HTMLParser import HTMLParserimport requestsclass MovieParser(HTMLParser):    def __init__(self):        HTMLParser.__init__(self)        self.movies=[]        self.in_movies = False    #重载父类方法    #循环处理feed进来的所有的tag,以及标签对应的attrs    def handle_starttag(self, tag, attrs):        #给定tag的属性名attrname，获取属性的值        def _attr(attrList,attrname):            for attr in attrList:                if attr[0] == attrname:                    return attr[1]            return None        if tag == 'li' and _attr(attrs,'data-title') \                and _attr(attrs,'data-category') == 'nowplaying':            movie = {}            movie['title'] = _attr(attrs,'data-title')            movie['score'] = _attr(attrs,'data-score')            movie['director'] = _attr(attrs,'data-director')            movie['actors'] = _attr(attrs,'data-actors')            self.movies.append(movie)            self.in_movies = True        if tag == 'img' and self.in_movies:            src = _attr(attrs,'src')            movie = self.movies[len(self.movies)-1]            #将电影的海报url添加到movie中            movie['poster-url'] = src            #根据海报的url下载图片            _download_poster_image(movie)            #下载完后将self.in_movies属性置为False            self.in_movies = False#根据海报图片的url下载海报图片def _download_poster_image(movie):    #"poster-url":"https://img3.doubanio.com/view/movie_poster_cover/mpst/public/p2354707516.jpg",    src = movie['poster-url']    r = requests.get(src)    fname = src.split('/')[-1]    #将图片下如文件    with open(fname,'wb') as f:        f.write(r.content)        movie['poster-path'] = fname#获取热播电影信息def nowplaying_movies(url):    headlers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'}    # req = urllib2.Request(url,headers=headlers)    # response = urllib2.urlopen(req)    # #定义解析器MovieParser继承自HTMLParser    # parser = MovieParser()    # #将response.read()喂给解析器，    # # 供解析器的handle_startendtag(self, tag, attrs)解析    # parser.feed(response.read())    # response.close()    # return parser.movies    #使用requests重构上面注释的代码    r = requests.get(url,headlers)    p = MovieParser()    p.feed(r.content)    return p.moviesif __name__ == '__main__':    url = 'https://movie.douban.com/nowplaying/qingdao/'    movies = nowplaying_movies(url)    #把movies以json格式打印出来    print '%s' % json.dumps(movies,sort_keys=True,indent=4,separators=(',',':'))