C#程序抓取网页实例

using System;using System.Collections.Generic;using System.ComponentModel;using System.Data;using System.Drawing;using System.Text;using System.Windows.Forms;using System.Net;using System.IO;using System.Text.RegularExpressions;using System.Collections;namespace CopyHtml{ public partial class Form1 : Form { public Form1() { InitializeComponent(); } private void button1_Click(object sender, EventArgs e) { //获取指定网页中的源数据 string rl; WebRequest Request = WebRequest.Create(textBox1.Text.Trim()); WebResponse Response = Request.GetResponse(); Stream resStream = Response.GetResponseStream(); StreamReader sr = new StreamReader(resStream, Encoding.Default); StringBuilder sb = new StringBuilder(); while ((rl = sr.ReadLine()) != null) { sb.Append(rl); } textBox2.Text = sb.ToString();//抓取得到的源网页 string he = textBox2.Text.ToString(); textBox3.Text = stripHtml(he);//去除html标签后得到的源网页 Match TitleMatch = Regex.Match(he, "<title>([^<]*)</title>", RegexOptions.IgnoreCase | RegexOptions.Multiline);//获取网页的标题 string title = TitleMatch.Groups[1].Value; textBox4.Text = ("网页的标题是：" + title ); } /// <summary> /// 去掉网页中的html标签 /// </summary> /// <param name="strHtml">待转化的字符串</param> /// <returns></returns> private string stripHtml(string strHtml) { Regex objRegExp = new Regex("<(.|/n)+?>"); string strOutput = objRegExp.Replace(strHtml, ""); strOutput = strOutput.Replace("<", "<"); strOutput = strOutput.Replace(">", ">"); return strOutput; } // 提取HTML代码中的网址 public static ArrayList GetHyperLinks(string htmlCode) { ArrayList al = new ArrayList(); string strRegex = @"(href)[ ]*=[ ]*[""'][^""'#>]+[""']"; Regex r = new Regex(strRegex, RegexOptions.IgnoreCase); MatchCollection m = r.Matches(htmlCode); for (int i = 0; i <= m.Count - 1; i++) { bool rep = false; string strNew = m[i].ToString(); // 过滤重复的URL foreach (string str in al) { if (strNew == str) { rep = true; break; } } if (!rep) al.Add(strNew); } al.Sort(); return al; } }}