python3 urllib 爬虫乱码问题解决

#!/usr/bin/env python# -*- coding: utf-8 -*-from bs4 import BeautifulSoupfrom urllib.request import urlopenbaseUrl = 'http://www.51bengou.com'def getCover(articleUrl):    global baseUrl    html = urlopen(baseUrl+articleUrl).read()    bsObj = BeautifulSoup(html, 'lxml')    try:        # Find internal link of the cover.        src = bsObj.find('div', {'class': 'cartoon-intro'}).find('img')['src']        return src    except AttributeError:        return Nonedef getInfo(articleUrl):    global baseUrl    html = urlopen(baseUrl+articleUrl)    bsObj = BeautifulSoup(html, 'lxml')    try:        # Find all information.        infos = bsObj.find('div', {'class': 'cartoon-intro'}).findAll('p', {'class': False})        items = {}        # Store in a dict.        for info in infos[:7]:            items[info.span.text] = info.text        return list(items)    except AttributeError:        return Noneprint(getInfo('/cartoon/HuoYingRenZhe/'))

如上程序是一个基于笨狗漫画网的爬虫程序，运行后，发现得到的漫画基本信息输出为乱码。

<meta charset="gb2312">

经查看网页源码发现，这个网页使用了gb2312编码。经我目前学习的编码知识，在程序读取网页时，BeautifulSoup使用了默认的utf-8编码将gb2312编码的字节字符串解码为了Unicode。此时，就出现了乱码，并且可能因为对错误的忽略或者替代，信息已经发生了丢失。

为了解决这个问题，我们应该在使用BeautifulSoup之前，对urlopen得到的对象进行读取，然后使用gb2312编码进行解码，此时问题应该就解决了。

#!/usr/bin/env python# -*- coding: utf-8 -*-from bs4 import BeautifulSoupfrom urllib.request import urlopenbaseUrl = 'http://www.51bengou.com'def getCover(articleUrl):    global baseUrl    html = urlopen(baseUrl+articleUrl).read().decode('gb2312', 'replace')    bsObj = BeautifulSoup(html, 'lxml')    try:        # Find internal link of the cover.        src = bsObj.find('div', {'class': 'cartoon-intro'}).find('img')['src']        return src    except AttributeError:        return Nonedef getInfo(articleUrl):    global baseUrl    html = urlopen(baseUrl+articleUrl).read().decode('gb2312', 'replace')    bsObj = BeautifulSoup(html, 'lxml')    print(html)    try:        # Find all information.        infos = bsObj.find('div', {'class': 'cartoon-intro'}).findAll('p', {'class': False})        items = {}        # Store in a dict.        for info in infos[:7]:            items[info.span.text] = info.text        return list(items)    except AttributeError:        return Noneprint(getInfo('/cartoon/HuoYingRenZhe/'))