抓取网页的两种方式.可直接使用

1: CloseableHttpClient 

/* * 爬取网页信息 */private static String pickData(String url) {CloseableHttpClient httpclient = HttpClients.createDefault();try {HttpGet httpget = new HttpGet(url);CloseableHttpResponse response = httpclient.execute(httpget);try {// 获取响应实体HttpEntity entity = response.getEntity();// 打印响应状态if (entity != null) {return EntityUtils.toString(entity);}} finally {response.close();}} catch (ClientProtocolException e) {e.printStackTrace();} catch (ParseException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {// 关闭连接,释放资源try {httpclient.close();} catch (IOException e) {e.printStackTrace();}}return null;}

2:模拟浏览器:

/* * 爬取网页信息 */public static String sendGet(String url) throws Exception {String result = "";URLConnection conn = null;InputStream inputStream = null;BufferedReader in = null;URL realURL = new URL(url);conn = realURL.openConnection();conn.setRequestProperty("accept", "*/*");conn.setRequestProperty("connection", "Keep-Alive");conn.setRequestProperty("user-agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");Map<String, List<String>> map = conn.getHeaderFields();for (String s : map.keySet()) {System.out.println(s + "-->" + map.get(s));}inputStream = conn.getInputStream();String line;// 读取操作in = new BufferedReader(new InputStreamReader(inputStream, "utf-8"));while ((line = in.readLine()) != null) {result += "\n" + line;}in.close();System.out.println(result);return result;}