JAVA抓取网站网页内容

最近在用JAVA研究下爬网技术,呵呵，入了个门,把自己的心得和大家分享下  
以下提供二种方法，一种是用apache提供的包．另一种是用JAVA自带的.  
代码如下:  
 
 
// 第一种方法  
//这种方法是用apache提供的包,简单方便  
//但是要用到以下包:commons-codec-1.4.jar  
//    commons-httpclient-3.1.jar  
//    commons-logging-1.0.4.jar  
 
 
    public static String createhttpClient(String url, String param) {  
        HttpClient client = new HttpClient();  
        String response = null;  
        String keyword = null;  
        PostMethod postMethod = new PostMethod(url);  
 
     
//        try {  
//            if (param != null)  
//                keyword = new String(param.getBytes("gb2312"), "ISO-8859-1");  
//        } catch (UnsupportedEncodingException e1) {  
//            // TODO Auto-generated catch block  
//            e1.printStackTrace();  
//        }  
 
        // NameValuePair[] data = { new NameValuePair("keyword", keyword) };  
        // // 将表单的值放入postMethod中  
        // postMethod.setRequestBody(data);  
        //    以上部分是带参数抓取,我自己把它注销了．大家可以把注销消掉研究下             
 
        try {  
            int statusCode = client.executeMethod(postMethod);  
            response = new String(postMethod.getResponseBodyAsString()  
                    .getBytes("ISO-8859-1"), "gb2312");//这里要注意下 gb2312要和你抓取网页的编码要一样  
              
            String p = response.replaceAll("\\&[a-zA-Z]{1,10};", "")  
                    .replaceAll("<[^>]*>", "");//去掉网页中带有html语言的标签  
            System.out.println(p);  
        } catch (Exception e) {  
 
            e.printStackTrace();  
        }  
        return response;  
 
    }  
 
    // 第二种方法  
    // 这种方法是JAVA自带的URL来抓取网站内容  
 
 
    public String getPageContent(String strUrl, String strPostRequest,  
            int maxLength) {  
        // 读取结果网页  
        StringBuffer buffer = new StringBuffer();  
        System.setProperty("sun.net.client.defaultConnectTimeout", "5000");  
        System.setProperty("sun.net.client.defaultReadTimeout", "5000");  
        try {  
            URL newUrl = new URL(strUrl);  
            HttpURLConnection hConnect = (HttpURLConnection) newUrl  
                    .openConnection();  
            // POST方式的额外数据  
            if (strPostRequest.length() > 0) {  
                hConnect.setDoOutput(true);  
                OutputStreamWriter out = new OutputStreamWriter(hConnect  
                        .getOutputStream());  
                out.write(strPostRequest);  
                out.flush();  
                out.close();  
            }  
            // 读取内容  
              
            BufferedReader rd = new BufferedReader(new InputStreamReader(  
                    hConnect.getInputStream()));  
            int ch;  
            for (int length = 0; (ch = rd.read()) > -1  
                    && (maxLength <= 0 || length < maxLength); length++)  
                buffer.append((char) ch);  
            String s = buffer.toString();  
            s.replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "");  
            System.out.println(s);  
              
            rd.close();  
            hConnect.disconnect();  
            return buffer.toString().trim();  
        } catch (Exception e) {  
            // return "错误:读取网页失败！";  
            //  
            return null;  
 
               
        }  
    }  
 
然后写个测试类:  
 
 
public static void main(String[] args) {  
 
        String url = "http://www.renren.com";  
        String keyword = "人人";  
        createhttpClient p = new createhttpClient();  
        String response = p.createhttpClient(url, keyword); // 第一种方法  
        // p.getPageContent(url, "post", 100500);//第二种方法  
    }  
 
呵呵，看看控制台吧,是不是把网页的内容获取了  
 
第三种方法：  
import java.io.FileOutputStream;  
import java.io.InputStream;  
import java.io.OutputStream;  
import java.net.URL;  
 
 
    public class GetUrlToHtml {  
        public static void main(String[] args) {  
            InputStream in = null;     
            OutputStream out = null;  
    
            try {  
 
 
 
 
                if ((args.length != 1)&& (args.length != 2))   
                    throw new IllegalArgumentException("Wrong number of args");  
           
                 
                URL url = new URL(args[0]);    
                in = url.openStream();          
                if (args.length == 2)           
                    out = new FileOutputStream(args[1]);  
                else out = System.out;  
           
      
 
 
 
 
                byte[] buffer = new byte[4096];  
                if(out==System.out){new String();}  
                int bytes_read;  
                while((bytes_read = in.read(buffer)) != -1){  
                    out.write(buffer, 0, bytes_read);}                       
    }  
            
            catch (Exception e) {  
                System.err.println(e);  
                System.err.println("Usage: java GetURL <URL> [<filename>]");  
            }  
            finally {   
                try { in.close(); out.close(); } catch (Exception e) {}  
            }  
        }  
    }