爬虫

package demo;import com.google.common.base.Joiner;import com.mongodb.BasicDBObject;import com.mongodb.DB;import com.mongodb.DBCollection;import com.mongodb.MongoClient;import org.jsoup.Connection;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;import redis.clients.jedis.Jedis;import java.io.IOException;import java.util.ArrayList;import java.util.List;import javax.swing.plaf.synth.SynthSpinnerUI;public class RedisSpider2 {    private static MongoClient mongo = new MongoClient( "localhost" , 27017 );    private static Jedis jedis=new Jedis("localhost");    /*    private static Queue celebrate_queue=new Queue();//大V队列    private static void InputSeedIntoQueue(){        celebrate_queue.enQueue("1782270602,286");        celebrate_queue.enQueue("yiwenmu,575");    }   */    public static String getSeedFromRedis() {        // 从Redis中获取数据并出队，uid以及connum，取出后处理分离，取出后即刻销毁        String element = jedis.lpop("follower:uid_connum");        // System.out.println("[DEBUG] : 当前使用了: " + element);        System.out.println("[DEBUG] : 现出队 -------------------- " + element);        return element;    }    public static void writeDataIntoMongo(BasicDBObject document){        MongoClient mongo;        mongo = new MongoClient( "localhost" , 27017 );        DB db = mongo.getDB("Belle");     //得到数据库        DBCollection table = db.getCollection("Following"); //拿到table        table.save(document);    }    public static void main(String[] args) throws IOException {         //InputSeedIntoQueue();        System.out.println("成功读取种子文件");        while(true){            String uid_fonum=celebrate_queue.deQueue();//读种子用户            System.out.println("[DEBUG] 大V "+uid_fonum+" 已出队列");            String[] parts = uid_fonum.split(",");//以逗号分隔开 存入parts字符串数组中            String uid = parts[0];  //分隔的第一个部分就是uid            int following = Integer.parseInt(parts[1]);  //把字符串转转换成整型            System.out.println("[DEBUG]----开始爬取-----"+uid_fonum+"-------");                     crawler_and_toMongo(uid,following);   //种子用户开始爬        }    }    private static void crawler_and_toMongo(String uid, int following) {        float total_number = (float) (following*1.0/32 +3);  //一页32个关注用户        ArrayList<String> certain_celebrate_list = new ArrayList();        for(int i=1;i<total_number;i++){            System.out.println("The current uid is "+uid+" and the page number is "+i);             String following_url = "http://tw.weibo.com/"+uid+"/follow/p/"+i;            Connection con = Jsoup.connect("http://tw.weibo.com/api/user/follow");//和url建立连接            con.data("uid",uid);                           con.data("page",String.valueOf(i));            con.data("currentuid","1991953413");              con.data("page_size","32");            //设了一个cookie 以新浪用户登录的            con.header("Cookie","SINAGLOBAL=9711952802808.682.1499668084041; __gads=ID=130480cbf75a5ad6:T=1500258878:S=ALNI_MZRcFnJveJoxtD4TBbJNbD__KcD-A; UM_distinctid=15d4e658b271f5-0573ee5fbac68c-1b1d7751-1fa400-15d4e658b28944; wvr=6; UOR=,,www.shejidaren.com; _ga=GA1.2.84290483.1500258858; _gid=GA1.2.952496907.1501988163; crtg_rta=; SSOLoginState=1502013392; SCF=Agl4g4wcoIanRFGqnVTCrYVRKcGcCSGCXik1olhn6h9oPJpHfvjbBZyqPuHU-ibuQXyHNclynoHBANDkf_WRtWc.; SUB=_2A250gpeCDeRhGedH4lMY9S3Nzz-IHXVX-Y5KrDV8PUNbmtBeLUWjkW9XZkSwKQAYTJkZTvK42M-ekE_Hnw..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWEbrh.i5bvbnSvvbQHi2M35JpX5KMhUgL.Fo241K24SKepShe2dJLoIpjLxKML1KBLBo5LxK-LB--LB-2LxK-L1h-L1h2t; SUHB=0LcjYQm8zbqhfQ; ALF=1533549388; _s_tentry=-; Apache=9574205823039.291.1502013327944; ULV=1502013327950:6:1:2:9574205823039.291.1502013327944:1501421673576");            con.header("Referer",following_url);  //关注            org.jsoup.nodes.Document doc;            try {                doc = con.post();                for (Element e: doc.getElementsByClass("fwBox") ){                    String following_url_redirect = e.attr("href");                    System.out.println(following_url_redirect);                    /////////////////////////去找每一个关注的人的粉丝数 判断其是不是大V                    Connection con_direct = Jsoup.connect(following_url_redirect);//和url建立连接                    con_direct.data("uid",uid);                                   con_direct.userAgent("Mozilla");                    con_direct.data("currentuid","1991953413");                      con_direct.data("page_size","32");                    //设了一个cookie 以新浪用户登录的                    con_direct.header("Cookie","SINAGLOBAL=9711952802808.682.1499668084041; __gads=ID=130480cbf75a5ad6:T=1500258878:S=ALNI_MZRcFnJveJoxtD4TBbJNbD__KcD-A; UM_distinctid=15d4e658b271f5-0573ee5fbac68c-1b1d7751-1fa400-15d4e658b28944; wvr=6; UOR=,,www.shejidaren.com; _ga=GA1.2.84290483.1500258858; _gid=GA1.2.952496907.1501988163; crtg_rta=; SSOLoginState=1502013392; SCF=Agl4g4wcoIanRFGqnVTCrYVRKcGcCSGCXik1olhn6h9oPJpHfvjbBZyqPuHU-ibuQXyHNclynoHBANDkf_WRtWc.; SUB=_2A250gpeCDeRhGedH4lMY9S3Nzz-IHXVX-Y5KrDV8PUNbmtBeLUWjkW9XZkSwKQAYTJkZTvK42M-ekE_Hnw..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWEbrh.i5bvbnSvvbQHi2M35JpX5KMhUgL.Fo241K24SKepShe2dJLoIpjLxKML1KBLBo5LxK-LB--LB-2LxK-L1h-L1h2t; SUHB=0LcjYQm8zbqhfQ; ALF=1533549388; _s_tentry=-; Apache=9574205823039.291.1502013327944; ULV=1502013327950:6:1:2:9574205823039.291.1502013327944:1501421673576");//                  con_direct.header("Referer","http://tw.weibo.com/libingbing/follow/p/12");//从李冰冰的微博关注名单的12页开始发送请求                    con_direct.header("Referer",following_url_redirect);  //关注                    org.jsoup.nodes.Document doc_direct;                try{                           doc_direct=con_direct.get();                   String   celebrant_following; //关注人数                   String   following_celebrant_uid;//大Vuid                    for (Element e1: doc_direct.select("#mInfo").select("ul").select("li.fansNum").select("a").select("strong")){                            String follower_num = e1.text();//抓取粉丝数                            if(Long.parseLong(follower_num)>1000000)//粉丝数如果大于100万  则为大V                            {                                //从url里截取uid                                following_celebrant_uid=following_url_redirect.substring(following_url_redirect.lastIndexOf("/")+1,following_url_redirect.length());                                //爬取去该大V的关注数                               Elements e2=doc_direct.select("#mInfo").select("ul").select("li.followNum").select("a").select("strong");                               celebrant_following=e2.text();                                //将新爬取的大V的uid和其关注数 连接成新的字符串                                String new_uid_fonum=following_celebrant_uid+","+celebrant_following;                                //将新爬取的大V的uid和其关注数 入队列                                celebrate_queue.enQueue(new_uid_fonum);                             System.out.println("[DEBUG]"+following_celebrant_uid+"   是大V且已经入队");                             certain_celebrate_list.add(following_celebrant_uid);                            }                        }                     String uids = Joiner.on(",").join(certain_celebrate_list);                        BasicDBObject document = new BasicDBObject();                        document.put("_id", uid); //该用户的uid                        document.put("following_celebrant", certain_celebrate_list);                        writeDataIntoMongo(document);                        //System.out.println("[DEBUG]"+following_celebrant_uid+"以入mongo");                    timeDelay(1,3);//最小为5 最大为15    单线程        } catch (IOException e1) {                // TODO Auto-generated catch block                e1.printStackTrace();            }                    }            }catch (IOException e1) {                // TODO Auto-generated catch block                //e1.printStackTrace();                timeDelay(30,35);                continue;            }        }    }    public static void timeDelay(float min, float max){        int random = (int)(max * Math.random() + min);        try {            Thread.sleep(random * 1000);//线程睡觉        } catch (InterruptedException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }}}