HTTP报文基本结构，自己写程序发送http包登陆某些网站。

          对http协议一直没有深入的研究，前几天寝室一哥们需要 登陆 某网站，批量查询一些客户的信息（大约有几万个客户），因为他嫌自己手动查询太慢（当然他也比较懒），所以找我帮他写一个程序，进行批量查询。刚开始他给我说的时候，我只是简单的说 叫他去用一用 按键精灵 这个软件，能够自动做重复的操作，于是 他听我的 去下载了一个按键精灵 玩了一玩，结果发现满足不了他的需求，并且他嫌按键精灵太慢了，太傻了。没办法 ，我只能给他写代码解决这个问题啦 ，思路很简单就是自己写程序与 某网站的服务器通信，然后写一个循环重复提交自己的客户信息，并解析返回的结果就ok了。由于对http不是非常熟悉，之前也没有类似的开发经历，于是我决定先看看http协议的基本结构，在网上看到一个图，觉得不错（具体链接地址：点击打开链接），要是对http不是很了解的可参考点击打开链接：

看完之后 ， 心里有点感觉 ，于是开始在网上摸索，于是找到了这样几个包：commons-codec-1.6.jar，commons-httpclient-3.0.1.jar，commons-logging-1.1.3.jar，jsoup-1.8.1.jar（所有的包下载地址：点击打开链接）

，前三个包是对http协议的封装，jsoup包是解析网页的包（jsoup中文参考资料：点击打开链接），在网上找了一个demo，然后自己就开始动手了，首先 你需要 去访你想要登陆的 网站去查看 其 登陆 过程，通过查看其 使用的是 get 还是 post 方法  ，当然一般的网站登陆都是使用（https+post），如果是使用https协议，你可能需要把 该网站上使用的证书下载下来（一会我会给出证书下载的代码），不然你和其进行通信是会提示证书错误的。初次访问某网站你能够 得到一个session 值，当然这个session值需要你解析网页获取（一般在返回的头部信息（set-Cookie属性中会有）），有些网站可能还会给你返回一个token（令牌），不过这个值是不会变的，这个值也可以通过解析网页获取到。总之，你需要在你的浏览器中登陆一遍某网站，然后你根据你的浏览器中登陆的过程，你自己写程序模拟浏览器的操作就ok啦，在登陆完成之后，你可能要保存一写你自己的私有信息（对浏览器来说就是cookie），比如，用户名，sessionid，token等，在之后的操作中，你只有给服务器传入这些你的私有信息，浏览器才知道你已经登陆了（一般在你的请求报头中的cookie信息中给出）。总结出来就几点：

1. 如果网站使用https协议，把该网站的证书下载下来 ，并在你的程序中设置一下证书。

2. 在浏览器中模拟登陆过程，并查看每次点击 之后 ，浏览器发送的报文信息。

3. 编写程序模拟浏览器的操作。

下载证书代码：在你的项目工程目录下会出现文件

jssecacerts，即为证书，下一个程序使用了此证书。

package com;/* * Copyright 2006 Sun Microsystems, Inc.  All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * *   - Redistributions of source code must retain the above copyright *     notice, this list of conditions and the following disclaimer. * *   - Redistributions in binary form must reproduce the above copyright *     notice, this list of conditions and the following disclaimer in the *     documentation and/or other materials provided with the distribution. * *   - Neither the name of Sun Microsystems nor the names of its *     contributors may be used to endorse or promote products derived *     from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */import java.io.BufferedReader;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.InputStream;import java.io.InputStreamReader;import java.io.OutputStream;import java.security.KeyStore;import java.security.MessageDigest;import java.security.cert.CertificateException;import java.security.cert.X509Certificate;import javax.net.ssl.SSLContext;import javax.net.ssl.SSLException;import javax.net.ssl.SSLSocket;import javax.net.ssl.SSLSocketFactory;import javax.net.ssl.TrustManager;import javax.net.ssl.TrustManagerFactory;import javax.net.ssl.X509TrustManager;public class InstallCert {public static void main(String[] args) throws Exception {String str[] = {"url"};//网址String host;int port;char[] passphrase;if ((str.length == 1) || (str.length == 2)) {String[] c = str[0].split(":");host = c[0];port = (c.length == 1) ? 443 : Integer.parseInt(c[1]);String p = (str.length == 1) ? "changeit" : str[1];passphrase = p.toCharArray();} else {System.out.println("Usage: java InstallCert <host>[:port] [passphrase]");return;}File file = new File("jssecacerts");if (file.isFile() == false) {char SEP = File.separatorChar;File dir = new File(System.getProperty("java.home") + SEP + "lib"+ SEP + "security");file = new File(dir, "jssecacerts");if (file.isFile() == false) {file = new File(dir, "cacerts");}}System.out.println("Loading KeyStore " + file + "...");InputStream in = new FileInputStream(file);KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());ks.load(in, passphrase);in.close();SSLContext context = SSLContext.getInstance("TLS");TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());tmf.init(ks);X509TrustManager defaultTrustManager = (X509TrustManager) tmf.getTrustManagers()[0];SavingTrustManager tm = new SavingTrustManager(defaultTrustManager);context.init(null, new TrustManager[] { tm }, null);SSLSocketFactory factory = context.getSocketFactory();System.out.println("Opening connection to " + host + ":" + port + "...");SSLSocket socket = (SSLSocket) factory.createSocket(host, port);socket.setSoTimeout(10000);try {System.out.println("Starting SSL handshake...");socket.startHandshake();socket.close();System.out.println();System.out.println("No errors, certificate is already trusted");} catch (SSLException e) {System.out.println();e.printStackTrace(System.out);}X509Certificate[] chain = tm.chain;if (chain == null) {System.out.println("Could not obtain server certificate chain");return;}BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));System.out.println();System.out.println("Server sent " + chain.length + " certificate(s):");System.out.println();MessageDigest sha1 = MessageDigest.getInstance("SHA1");MessageDigest md5 = MessageDigest.getInstance("MD5");for (int i = 0; i < chain.length; i++) {X509Certificate cert = chain[i];System.out.println(" " + (i + 1) + " Subject "+ cert.getSubjectDN());System.out.println("   Issuer  " + cert.getIssuerDN());sha1.update(cert.getEncoded());System.out.println("   sha1    " + toHexString(sha1.digest()));md5.update(cert.getEncoded());System.out.println("   md5     " + toHexString(md5.digest()));System.out.println();}System.out.println("Enter certificate to add to trusted keystore or 'q' to quit: [1]");String line = reader.readLine().trim();int k;try {k = (line.length() == 0) ? 0 : Integer.parseInt(line) - 1;} catch (NumberFormatException e) {System.out.println("KeyStore not changed");return;}X509Certificate cert = chain[k];String alias = host + "-" + (k + 1);ks.setCertificateEntry(alias, cert);OutputStream out = new FileOutputStream("jssecacerts");ks.store(out, passphrase);out.close();System.out.println();System.out.println(cert);System.out.println();System.out.println("Added certificate to keystore 'jssecacerts' using alias '"+ alias + "'");}private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();private static String toHexString(byte[] bytes) {StringBuilder sb = new StringBuilder(bytes.length * 3);for (int b : bytes) {b &= 0xff;sb.append(HEXDIGITS[b >> 4]);sb.append(HEXDIGITS[b & 15]);sb.append(' ');}return sb.toString();}private static class SavingTrustManager implements X509TrustManager {private final X509TrustManager tm;private X509Certificate[] chain;SavingTrustManager(X509TrustManager tm) {this.tm = tm;}public X509Certificate[] getAcceptedIssuers() {throw new UnsupportedOperationException();}public void checkClientTrusted(X509Certificate[] chain, String authType)throws CertificateException {throw new UnsupportedOperationException();}public void checkServerTrusted(X509Certificate[] chain, String authType)throws CertificateException {this.chain = chain;tm.checkServerTrusted(chain, authType);}}}

post协议实现代码：

public void init(String url,int count){System.setProperty("javax.net.ssl.trustStore", "jssecacerts"); //设置证书        PostMethod postMethod = new PostMethod(url); 

<span style="white-space:pre"></span>//这些信息根据你在浏览器中，模拟登陆是看到的报头来设置        postMethod.setRequestHeader("Accept", "text/html, application/xhtml+xml, */*");        postMethod.setRequestHeader("Accept-Language", "zh-CN");       // postMethod.setRequestHeader("Accept-Encoding", "gzip, deflate");

<span style="white-space:pre"></span>//不设置<span style="font-family: Arial, Helvetica, sans-serif;">Accept-Encoding</span>的话，不需要对网页进行gzip解压，但是流量会大一些       //postMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko");        postMethod.setRequestHeader("Host", "192.168.1.1");        postMethod.setRequestHeader("DNT", "1");        postMethod.setRequestHeader("Connection", "Keep-Alive");        postMethod.setRequestHeader("Cache-Control", "no-cache");        int statusCode = 0;          try {              statusCode = httpClient.executeMethod(postMethod);          } catch (HttpException e) {              e.printStackTrace();          } catch (IOException e) {              e.printStackTrace();          }         if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY                  || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {              // 从头中取出转向的地址              Header locationHeader = postMethod.getResponseHeader("location");              String location = null;              if (locationHeader != null) {                  location = locationHeader.getValue();                  System.out.println("The page was redirected to:" + location);                if ( count <= 10 )                {                init(location,count+1);//用跳转后的页面重新请求。                }                else                {                return;                }            } else {                  System.err.println("Location field value is null.");              }          } else {              try {              token = ParsePage.getToken(postMethod.getResponseBodyAsString());            if ( token == null || token.length() == 0 )            {            if ( count <= 10 )            init(url,count+1);//重新初始化，获取token            return;            }            System.out.println("token : " + token);            if ( postMethod.getResponseHeader("Set-Cookie") != null &&             postMethod.getResponseHeader("Set-Cookie").getValue().length() > 0)            {            String temp = postMethod.getResponseHeader("Set-Cookie").getValue();            jsessionid = temp.substring(temp.indexOf('=') + 1,temp.indexOf(';'));//获取session id                System.out.println("jsessionid : " + jsessionid);            //System.out.println(temp + "   " + jssesionid);            }            else            {            if ( count <= 10 )            init(url,count+1);//重新初始化，获取session id            return;            }            } catch (IOException e) {                  e.printStackTrace();              }              //postMethod.releaseConnection();          }  }

因为我写的程序涉及到一些隐私，就不全部给出了，关于其他的使用方式，读者自己摸索吧，如果有什么问题我希望能与各位读者一起探讨。