利用python抓取搜狗关于数据分析的文章并保存到csv文件

#coding = 'utf-8'          #这里很重要,如果不输入会发现储存的数据乱码,我是python3.6.1  import requestsfrom urllib.parse import urlencodefrom requests.exceptions import ConnectionErrorfrom pyquery import PyQuery as pqimport csvbase_url = 'http://weixin.sogou.com/weixin?'headers={    'Cookie': '这里要登陆然后复制你的cookie，在审查元素的network里',    'Host': 'weixin.sogou.com',    'Referer': 'http://weixin.sogou.com/weixin?type=2&query=%E9%A3%8E%E6%99%AF&ie=utf8&s_from=input&_sug_=n&_sug_type_=1&w=01015002&oq=&ri=1&sourceid=sugg&sut=0&sst0=1499651269966&lkt=0%2C0%2C0&p=40040108',    'Upgrade-Insecure-Requests':'1',    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}keyword = '数据分析'def get_html(url):    try:        response = requests.get(url,allow_redirects=False,headers=headers)   #当出现302时默认会进行跳转，allow_redirects=False是阻止其进行跳转的        if response.status_code == 200:            return response.text        if response.status_code == 302:            print('302')    except ConnectionError:        return get_html(url)def get_index(keyword,page):    data={        'query' : keyword,        'page': page,        'type': 2    }    queries = urlencode(data)    url = base_url + queries    html = get_html(url)    return htmldef parse_index(html):    doc = pq(html)    items = doc('.news-box li .txt-box h3 a').items()    for item in items:        yield item.attr('href')def get_detail(url):    try:        response = requests.get(url)        if response.status_code == 200:            return response.text        return None    except ConnectionError:        return Nonedef parse_detail(html):    doc = pq(html)    title = doc('.rich_media_title').text()    content = doc('.rich_media_content ').text()    date = doc('#post-date').text()    wechat = doc('#post-user').text()    return [title,content,date,wechat]def main():    with open('E:/weixin.csv', 'w', encoding='utf8', newline='') as f:  #在这里利用python自带的csv模块把数据保存到当地的文件夹下，可以指定文件目录，我这里是E盘        csv_writer = csv.writer(f, dialect='excel')        for page in range(1,11):                html = get_index(keyword,page)                article_urls = parse_index(html)                if html:                    for article_url in article_urls:                        article_html = get_detail(article_url)                        if article_html:                            article_data = parse_detail(article_html)                            csv_writer.writerow(article_data)                            print(article_data)if __name__ == '__main__':    main()

基本方法就是先获取搜狗微信文章索引目录的url，然后根据这个url进行深度爬取， 把微信文章、作者、时间、之类全部爬下来

储存的数据如下图所示