java用正则表达式分析读取网页内容
来源:互联网 发布:飒漫画软件下载 编辑:程序博客网 时间:2024/06/15 06:40
package com.xiaofeng.picup;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;import java.net.MalformedURLException;import java.net.URL;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.regex.Matcher;import java.util.regex.Pattern;/** *//** * * @author 晓峰2007.1.18 抓取雅虎知识堂的文章标题及内容(测试) 手动输入网址抓取,可进一步自动抓取整个知识堂的全部内容 * */public class WebContent ...{ /** *//** * 读取一个网页全部内容 */ public String getOneHtml(String htmlurl) throws IOException...{ URL url; String temp; StringBuffer sb = new StringBuffer(); try ...{ url = new URL(htmlurl); BufferedReader in = new BufferedReader(new InputStreamReader(url .openStream(), "utf-8"));// 读取网页全部内容 while ((temp = in.readLine()) != null) ...{ sb.append(temp); } in.close(); }catch(MalformedURLException me)...{ System.out.println("你输入的URL格式有问题!请仔细输入"); me.getMessage(); throw me; }catch (IOException e) ...{ e.printStackTrace(); throw e; } return sb.toString(); } /** *//** * * @param s * @return 获得网页标题 */ public String getTitle(String s) ...{ String regex; String title = ""; List<String> list = new ArrayList<String>(); regex = "<title>.*?</title>"; Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } for (int i = 0; i < list.size(); i++) ...{ title = title + list.get(i); } return outTag(title); } /** *//** * * @param s * @return 获得链接 */ public List<String> getLink(String s) ...{ String regex; List<String> list = new ArrayList<String>(); regex = "<a[^>]*href=("([^"]*)"|'([^']*)'|([^s>]*))[^>]*>(.*?)</a>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 获得脚本代码 */ public List<String> getScript(String s) ...{ String regex; List<String> list = new ArrayList<String>(); regex = "<script.*?</script>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 获得CSS */ public List<String> getCSS(String s) ...{ String regex; List<String> list = new ArrayList<String>(); regex = "<style.*?</style>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 去掉标记 */ public String outTag(String s) ...{ return s.replaceAll("<.*?>", ""); }
//**
*
* @param s
* @return 获取雅虎知识堂文章标题及内容
*/
public HashMap<String, String> getFromYahoo(String s) ...{
HashMap<String, String> hm = new HashMap<String, String>();
StringBuffer sb = new StringBuffer();
String html="";
System.out.println(" ------------------开始读取网页(" + s
+ ")--------------------");
try...{
html= getOneHtml(s);
}catch(Exception e)...{
e.getMessage();
}
// System.out.println(html);
System.out.println("------------------读取网页(" + s
+ ")结束-------------------- ");
System.out.println("------------------分析(" + s
+ ")结果如下-------------------- ");
String title = outTag(getTitle(html));
title = title.replaceAll("_雅虎知识堂", "");
// Pattern pa=Pattern.compile("<div
// class="original">(.*?)(( )*)(.*?)(( )*)(.*?)</div>",Pattern.DOTALL);
Pattern pa = Pattern.compile("<div class="original">(.*?)</p></div>",
Pattern.DOTALL);
Matcher ma = pa.matcher(html);
while (ma.find()) ...{
sb.append(ma.group());
}
String temp = sb.toString();
temp = temp.replaceAll("(<br>)+?", " ");// 转化换行
temp = temp.replaceAll("<p><em>.*?</em></p>", "");// 去图片注释
hm.put("title", title);
hm.put("original", outTag(temp));
return hm;
}
/** *//**
*
* @param args
* 测试一组网页,针对雅虎知识堂
*/
public static void main(String args[]) ...{
String url="";
List<String> list = new ArrayList<String>();
System.out.print("输入URL,一行一个,输入结束后输入 go 程序开始运行: ");
/**//*
http://ks.cn.yahoo.com/question/1307121201133.html
http://ks.cn.yahoo.com/question/1307121101907.html
http://ks.cn.yahoo.com/question/1307121101907_2.html
http://ks.cn.yahoo.com/question/1307121101907_3.html
http://ks.cn.yahoo.com/question/1307121101907_4.html
http://ks.cn.yahoo.com/question/1307121101907_5.html
http://ks.cn.yahoo.com/question/1307121101907_6.html
http://ks.cn.yahoo.com/question/1307121101907_7.html
http://ks.cn.yahoo.com/question/1307121101907_8.html
*/
BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
try ...{
while(!(url=br.readLine()).equals("go"))...{
list.add(url);
}
} catch (Exception e) ...{
e.getMessage();
}
WebContent wc = new WebContent();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < list.size(); i++) ...{
hm = wc.getFromYahoo(list.get(i));
System.out.println("标题: " + hm.get("title"));
System.out.println("内容: " + hm.get("original"));
}
/**//*
String htmlurl[] = {
"http://ks.cn.yahoo.com/question/1307121201133.html",
"http://ks.cn.yahoo.com/question/1307121101907.html",
"http://ks.cn.yahoo.com/question/1307121101907_2.html",
"http://ks.cn.yahoo.com/question/1307121101907_3.html",
"http://ks.cn.yahoo.com/question/1307121101907_4.html",
"http://ks.cn.yahoo.com/question/1307121101907_5.html",
"http://ks.cn.yahoo.com/question/1307121101907_6.html",
"http://ks.cn.yahoo.com/question/1307121101907_7.html",
"http://ks.cn.yahoo.com/question/1307121101907_8.html" };
WebContent wc = new WebContent();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < htmlurl.length; i++) {
hm = wc.getFromYahoo(htmlurl[i]);
System.out.println("标题: " + hm.get("title"));
System.out.println("内容: " + hm.get("original"));
}
*/
/**//*
* String html=""; String link=""; String sscript=""; String content="";
* System.out.println(htmlurl+" 开始读取网页内容:");
* html=wc.getOneHtml(htmlurl); System.out.println(htmlurl+"
* 读取完毕开始分析……"); html=html.replaceAll("(<script.*?)(( )*)(.*?)(( )*)(.*?)(</script>)","
* ");//去除脚本 html=html.replaceAll("(<style.*?)(( )*)(.*?)(( )*)(.*?)(</style>)","
* ");//去掉CSS html=html.replaceAll("<title>.*?</title>"," ");//除去页面标题
* html=html.replaceAll("<a[^>]*href=("([^"]*)"|'([^']*)'|([^s>]*))[^>]*>(.*?)</a>","
* ");//去掉链接 html=html.replaceAll("(s){2,}?"," ");//除去多余空格
* html=wc.outTag(html);//多余标记 System.out.println(html);
*/
/**//*
* String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
* content=(content.length()>s[i].length())?content:s[i]; }
* System.out.println(content);
*/
// System.out.println(htmlurl+"网页内容结束");
/**//*
* System.out.println(htmlurl+"网页脚本开始:"); List
* script=wc.getScript(html); for(int i=0;i<script.size();i++){
* System.out.println(script.get(i)); }
* System.out.println(htmlurl+"网页脚本结束:");
*
* System.out.println(htmlurl+"CSS开始:"); List css=wc.getCSS(html);
* for(int i=0;i<css.size();i++){ System.out.println(css.get(i)); }
* System.out.println(htmlurl+"CSS结束:");
*
* System.out.println(htmlurl+"全部链接内容开始:"); List list=wc.getLink(html);
* for(int i=0;i<list.size();i++){ link=list.get(i).toString(); }
* System.out.println(htmlurl+"全部链接内容结束:");
*
* System.out.println("内容"); System.out.println(wc.outTag(html));
*/
}
}
- java用正则表达式分析读取网页内容(1)
- java用正则表达式分析读取网页内容(2)
- java用正则表达式分析读取网页内容
- java读取(正则表达式分析)网页内容
- java读取(正则表达式分析)网页内容
- 正则表达式抽取网页内容
- java读取网页内容
- java根据 正则表达式解析html网页内容
- 用正则表达式获取网页里面的内容/java面试题
- java网页正则表达式爬虫(用纯java代码在相应的网页中查看自己想要的内容)
- 正则表达式与网页分析
- 正则表达式分析网页数据
- 用正则表达式对网页进行有效内容抽取
- 用正则表达式提取网页上表格的内容
- 用正则表达式对网页进行有效内容抽取
- 【Java】读取网页中的内容
- java 读取网页内容代码
- 网页内容提取常用正则表达式
- vsftpd主动、被动模式配置笔记
- MTK Android Driver :lcm
- 信号量
- Android的ADB工具使用
- Android Activity 生命周期的透彻理解
- java用正则表达式分析读取网页内容
- MTK Android Driver :memory
- 鸟哥谈PHP的架构与未来发展
- 多线程
- 【live555】ServerMediaSubsession具体与流有关的操作和参数
- java class反编译后的代码还原
- 逃出你的肖申克(一):为什么一定要亲身经历了之后才能明白?
- linux内存清理和释放命令
- TCP/IP与Socket (转)