htmlagilitypackDemo

using System;using System.Collections.Generic;using System.Linq;using System.Text;using HtmlAgilityPack;using System.Net;using System.IO;namespace htmlagilitypackDemo{    class Program    {        private const string CategoryListXPath = "./div[2]/div"; //关键点，不同网站分析不同的路径        private const string CategoryNameXPath = "./table[1]/tr[1]/td[2]/h3[1]"; //关键点，不同网站分析不同的路径        private const string ChooseXPath = "./a[1]";        static void Main(string[] args)        {            Demo_1();            Console.Read();        }        static void Demo_1()        {            Uri url = new Uri("http://news.sogou.com/news?query=%BA%A3%B6%FB&p=42230305&dp=1&_ast=1353663787&_asf=news.sogou.com&time=0&w=03009900&sort=1&mode=1&manual=");            Uri uriCategory = null;            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);            WebResponse response = request.GetResponse();            Stream stream = response.GetResponseStream();            StreamReader read = new StreamReader(stream, Encoding.GetEncoding("gb2312"));            string str = read.ReadToEnd();            HtmlDocument html = new HtmlDocument();            html.LoadHtml(str);            //HtmlNode rootNode = html.DocumentNode;            HtmlNode rootNode = html.GetElementbyId("main");            HtmlNodeCollection categoryNodeList = rootNode.SelectNodes(CategoryListXPath);            HtmlNode temp = null;            var list = new List<Category>();            foreach (HtmlNode categoryNode in categoryNodeList)            {                temp = HtmlNode.CreateNode(categoryNode.OuterHtml);                HtmlNode singleNode = temp.SelectSingleNode(CategoryNameXPath);                if (singleNode == null)                    continue;                HtmlNodeCollection singleList = temp.SelectNodes(CategoryNameXPath);                foreach (HtmlNode node in singleList)                {                    HtmlNode createNode = HtmlNode.CreateNode(node.OuterHtml);                    HtmlNode reNode = createNode.SelectSingleNode(ChooseXPath);                    if (reNode == null)                        continue;                    Category category = new Category();                    category.Subject = reNode.InnerText;                    Uri.TryCreate(url, reNode.Attributes["href"].Value, out uriCategory);                    category.IndexUrl = uriCategory.ToString();                    list.Add(category);                    Console.WriteLine("name:{0}",category.Subject);                    Console.WriteLine("url:{0}",category.IndexUrl);                    //break;                }            }        }    }    public class Category    {        public string Subject { get; set; }        public string IndexUrl { get; set; }    }}