模拟web访问有登录且有验证码的登录后抓取数据

1 取验证码

1 在窗体上放一个picturebox (imgValidate)存放获取的验证码图片，
2 用浏览器的开发者工具firefox (f12) 分析出验证码的网址
private void GetValidateImage()
        {
            cookies = new CookieContainer();
            string strUrl = "http://www.xxx.com/ValidateCodePicture.aspx?Key="+strValidCode;  //验证码页面 strValidCode这个随机码要先取出来

            CookieContainer cc = new CookieContainer();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);

            //set request args
            request.Method = "Get";
            request.CookieContainer = cc;
            request.KeepAlive = true;

            //request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
            request.ContentType = "text/html";

            //模拟goole浏览器访问
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
            //request.Referer = strUrl;
            request.Headers.Add("x-requested-with:XMLHttpRequest");
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en;q=0.6,nl;q=0.4,zh-TW;q=0.2");
            //request.ContentLength = postdataByte.Length;  text/html; charset=utf-8
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip |
                                             DecompressionMethods.None;
            //支持跳转页面，查询结果将是跳转后的页面
            ////request.AllowAutoRedirect = true;

            request.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (request.Method == "POST")
            {
                (request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
            }

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            MemoryStream ms = null;
            using (var stream = response.GetResponseStream())
            {
                Byte[] buffer = new Byte[response.ContentLength];
                int offset = 0, actuallyRead = 0;
                do
                {
                    actuallyRead = stream.Read(buffer, offset, buffer.Length - offset);
                    offset += actuallyRead;
                }
                while (actuallyRead > 0);
                ms = new MemoryStream(buffer);
            }
            response.Close();

            cookies = request.CookieContainer; //保存cookies
            strCookies = request.CookieContainer.GetCookieHeader(request.RequestUri); //把cookies转换成字符串

            Bitmap sourcebm = new Bitmap((Stream)ms);//初始化Bitmap图片
            imgValidate.Image = sourcebm;
        }

2 取js赋值的内容

  有的网页用查看网页源代码的方式看不到控件的值，需要用到下面的方法
  即用C#自带的webbrowse来加载网页，再用webBrowser1.Document来取对应控件的值，如
  tring strMsg2 = webBrowser1.Document.GetElementById("hdValidateCodeID").OuterHtml;

3 取得要提交的参数

  如果是asp.net的网页还有提交”__EVENTTARGET“,"__EVENTARGUMENT","__VIEWSTATE"这三个参数，这个也可以在开发者工具-网络-参数里看到
可以用httpRequest先取得源代码再分析出
这里用的是webbrowse里加载好的
  
        private void GetViewState()
        {
            string strMsg = webBrowser1.Document.GetElementById("__VIEWSTATE").OuterHtml;
            //取viewstate value
            //<INPUT id=__VIEWSTATE type=hidden value=/wEPDwUKMTg0NTk3Mjg2N2Rk name=__VIEWSTATE>
            MatchCollection mc = Regex.Matches(strMsg, "id=__VIEWSTATE.*(?<viewstate>value[^>]*)", RegexOptions.IgnoreCase);

            if (mc.Count > 0)
            {
                foreach (Match m in mc)
                {
                    strViewState = m.Groups["viewstate"].Value.ToString().Trim();
                    if (strViewState.Length > 0)
                    {
                        strViewState = strViewState.Replace("value=", "").Replace("\"", "").Replace("\\", "").Replace("name=__VIEWSTATE","").Replace(" ","");
                    }
                }
            }

            //<INPUT id=hdValidateCodeID type=hidden value=c1b52d3a-1f8b-1dc4-0d44-32a4b46ef8af name=hdValidateCodeID>
            string strMsg2 = webBrowser1.Document.GetElementById("hdValidateCodeID").OuterHtml;
            MatchCollection mc2 = Regex.Matches(strMsg2, "id=hdValidateCodeID.*(?<validatecode>value[^>]*)", RegexOptions.IgnoreCase);

            if (mc2.Count > 0)
            {
                foreach (Match m in mc2)
                {
                    strValidCode = m.Groups["validatecode"].Value.ToString().Trim();
                    if (strValidCode.Length > 0)
                    {
                        strValidCode = strValidCode.Replace("value=", "").Replace("\"", "").Replace("\\", "").Replace("/", "").Replace("name=hdValidateCodeID","").Replace(" ","");
                    }
                }
            }
            txtValidCode.Text = strValidCode;
            txtViewState.Text = strViewState;

            //String 的Cookie　要转成　Cookie型的　并放入CookieContainer中  
            string cookieStr = webBrowser1.Document.Cookie;
            string[] cookstr = cookieStr.Split(';');

            foreach (string str in cookstr)
            {
                try
                {
                    string[] cookieNameValue = str.Split('=');
                    Cookie ck = new Cookie(cookieNameValue[0].Trim().ToString(), cookieNameValue[1].Trim().ToString());
                    ck.Domain = "XXX.com"; //必须写对  
                    myCookieContainer.Add(ck);
                }
                catch
                {
                }
            }  
        }
  

4 登录并且存取cookie

提交参数，并存下cookie,供后续用
private void Login()
        {
            cookies = new CookieContainer();
            string strUrl = "http://www.xxx.com/Login.aspx";  //验证码页面

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);

            //set request args
            request.Method = "POST";
            request.CookieContainer = myCookieContainer;
            request.KeepAlive = true;

            //request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
            request.ContentType = "text/html";


            //模拟goole浏览器访问
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
            //request.Referer = strUrl;
            request.Headers.Add("x-requested-with:XMLHttpRequest");
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en;q=0.6,nl;q=0.4,zh-TW;q=0.2");
            //request.ContentLength = postdataByte.Length;  text/html; charset=utf-8
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip |
                                             DecompressionMethods.None;
            //支持跳转页面，查询结果将是跳转后的页面
            ////request.AllowAutoRedirect = true;

            request.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (request.Method == "POST")
            {
                (request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
            }

            //---begin
           
            string postData = string.Format("txtUserName={0}&txtPassword={1}&txtValidateCode={2}&hdValidateCodeID={3}&ddlLanguage=CN&btnLogin=登录&__EVENTTARGET=&__EVENTARGUMENT=&__VIEWSTATE={4}", txtUserName.Text, txtPassword.Text, txtValidate.Text,strValidCode,strViewState);  //这里按照前面FireBug中查到的POST字符串做相应修改。
            byte[] postdatabyte = Encoding.UTF8.GetBytes(postData);
           
            request.ContentLength = postdatabyte.Length;

            using (Stream stream = request.GetRequestStream())
            {
                stream.Write(postdatabyte, 0, postdatabyte.Length);
            }
            //---end---

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            //StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gb2312"));
            StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
            string strMsg = reader.ReadToEnd();

            response.Close();

            cookies = request.CookieContainer; //保存cookies,后面再请求其它网页就可用这个cookie,不用在登录了
            lbLogin.Text = "已登录";
            btnSearchResume.Enabled = true;

        }