简书首页推荐文章文字爬取，用txt保存

来源：互联网发布：移动网络投诉处理编辑：程序博客网时间：2024/04/30 18:09

发现简书上有些文章还挺不错，页面如下：

简书页面

然后就手痒写了几行代码，用xpath匹配的方法将首页推荐的文章的内容抓了下来，它有一个“显示更多的按钮”，每次click一下，就会再出现一些内容，这次我设置的是抓了4次更多。

点击查看更多

之后就是保存在本地了，代码不多，但是还算实用，只是没有将文章里的图片给配套爬取。

详细结果

代码如下：

package qita;import java.io.IOException;import java.util.Vector;import java.io.FileWriter;import com.hpre.spider.common.Function;import com.hpre.spider.tools.Tools;public class Download_jianshu {    public static void main(String[] args) throws IOException {        String url = "http://www.jianshu.com/";        byte[] information = Function.download(url);//      System.out.println(new String(information));        Vector<String> hrefs = new Vector<String>();        Vector<String> title = new Vector<String>();        Vector<String> tiaoye = new Vector<String>();        FileWriter writer;        Tools.getMultiResultsByOneXpathPattern(information, "utf-8",                "//div[@class='load-more']/button/@data-url", tiaoye);        Tools.getMultiResultsByOneXpathPattern(information, "utf-8",                "//div[@id='list-container']//li[@class='have-img']/div/h4/a/text()", title);        Tools.getMultiResultsByOneXpathPattern(information, "utf-8",                "//div[@id='list-container']//li[@class='have-img']/div/h4/a/@href", hrefs);        for (int j = 0 ;j<tiaoye.size()&&j<=4;j++){            String page_url = url + tiaoye.get(j);            byte[] more_information = Function.download(page_url);            Tools.getMultiResultsByOneXpathPattern(more_information, "utf-8",                    "//div[@class='load-more']/button/@data-url", tiaoye);            Tools.getMultiResultsByOneXpathPattern(more_information, "utf-8",                    "//div[@id='list-container']//li[@class='have-img']/div/h4/a/text()", title);            Tools.getMultiResultsByOneXpathPattern(more_information, "utf-8",                    "//div[@id='list-container']//li[@class='have-img']/div/h4/a/@href", hrefs);        }        for (int i = 0;i<hrefs.size();i++){            Vector<String> content = new Vector<String>();            String new_href = url + hrefs.get(i);            byte[] detail_information = Function.download(new_href);            Tools.getMultiResultsByOneXpathPattern(detail_information, "utf-8",                    "//div[@class='show-content']//p/text()", content);            System.out.println(content.toString());            writer = new FileWriter("C://Users//Administrator//Desktop//jianshu//"+title.get(i).trim().replace("|", "")                    .replace("?", "")+".txt");            for (int n = 0;n<content.size();n++){                String lines = content.get(n).trim() + "\r\n";                    writer.write(lines);            }            writer.flush();            writer.close();        }    }}

0 0