利用正则表达式抽取网页信息

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class ObtainNews {


/**
* 
* @param htmlurl
* @return 读取一个网页全部内容
* @throws IOException
*/
public String getHtml(final String htmlurl) throws IOException {
URL url;
String temp;
final StringBuffer sb = new StringBuffer();
try {
url = new URL(htmlurl);
final BufferedReader in = new BufferedReader(new InputStreamReader(
url.openStream(), "utf-8"));// 读取网页全部内容
while ((temp = in.readLine()) != null) {
sb.append(temp);
}
in.close();
} catch (final MalformedURLException me) {
System.out.println("你输入的URL格式有问题！请仔细输入");
me.getMessage();
throw me;
} catch (final IOException e) {
e.printStackTrace();
throw e;
}
return sb.toString();
}


/**
* 获取 标题/正文/发布时间/发布者：null/来源站点：reuters/记者/分类频道/专题标识：null/图片/视频
*/


public String getTitle(final String s) {
String regex;
String title = "";
final List<String> list = new ArrayList<String>();
regex = "<h1>.*?</h1>";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
title = title + list.get(i);
}
return title;
}

public String getContent(final String s) {
String regex;
String content = "";
final List<String> list = new ArrayList<String>();
regex = "<span id=\"midArticle_start\"></span>.*?</span></span>";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
content = content + list.get(i);
}
return content;
}

public String getTime(final String s) {
String regex;
String time = "";
final List<String> list = new ArrayList<String>();
regex = "<span class=\"timestamp\">.*?</span>        </p>";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
time = time + list.get(i);
}
return time;
}

public String getReporter(final String s) {
String regex;
String reporter = "";
final List<String> list = new ArrayList<String>();
regex = "<p class=\"byline\">.*?</p>        <p>";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
reporter = reporter + " " + list.get(i);
}
return reporter;
}

public String getChannel(final String s) {
String regex;
String channel = "";
final List<String> list = new ArrayList<String>();
regex = "<div class=\"actionButton\">.*?</a></div>";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
channel = channel + " " + list.get(i);
}
return channel;
}

public String getImgsrc(final String s) {
String regex;
String imgsrc = "";
final List<String> list = new ArrayList<String>();
regex = "<img        src=\".*?\"        border";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
imgsrc = imgsrc + list.get(i);
}
return imgsrc;
}

public String getVideosrc(final String s) {
String regex;
String videosrc = "";
final List<String> list = new ArrayList<String>();
regex = "<div class=\"photo\">.*?<img";
final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find()) {
list.add(ma.group());
}
for (int i = 0; i < list.size(); i++) {
videosrc = videosrc + list.get(i);
}
return videosrc;
}

/**
* 
* @param s
* @return 去除标签
*/

public String outTag(final String s) {
return s.replaceAll("<.*?>", "");
}


/**
* 
* @param s
* @return 获取内容
*/


public HashMap<String, String> getFromWeb(final String s) {

final HashMap<String, String> hm = new HashMap<String, String>();
String html = "";
System.out.println("\n开始读取网页(" + s + ")");
try {
html = getHtml(s);
} catch (final Exception e) {
e.getMessage();
}
System.out.println(html);
System.out.println("分析(" + s + ")结果\n");
String title = outTag(getTitle(html));
String content = outTag(getContent(html));
String time = outTag(getTime(html));
String reporter = outTag(getReporter(html)).replaceAll("By ", "");
String channel = outTag(getChannel(html));
String imgsrc = getImgsrc(html)
.replaceAll("<img        src=\"", "").replaceAll("\"        border", "").replaceAll(" ", "");
String videosrc = getVideosrc(html)
.replaceAll("<div class=\"photo\"><a href=\"", "").replaceAll("\"><img", "");
hm.put("title", title);
hm.put("content", content);
hm.put("time", time);
hm.put("reporter", reporter);
hm.put("channel", channel);
hm.put("imgsrc", imgsrc);
hm.put("videosrc", videosrc);
return hm;


}


/**
* 
* @param args
*            测试网页www.reuters.com
*/


public static void main(final String args[]) {

String url = "";
final List<String> list = new ArrayList<String>();
System.out.print("输入新闻页面网址，换行输入run\n");
final BufferedReader br = new BufferedReader(new InputStreamReader(
System.in));
//http://www.reuters.com/article/2014/04/04/us-usa-cia-interrogation-idUSBREA321UC20140404
//http://www.reuters.com/article/2014/04/04/us-congress-justice-highspeed-idUSBREA3310O20140404
try {
while (!(url = br.readLine()).equals("run")) {
list.add(url);
}
} catch (final Exception e) {
e.getMessage();
}
final ObtainNews on = new ObtainNews();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < list.size(); i++) {
hm = on.getFromWeb(list.get(i));
String title = hm.get("title");
String content = hm.get("content");
String time = hm.get("time");
String publisher = null;
String site = "reuters";
String reporter = hm.get("reporter"); if(reporter == "") reporter = null;
String channel = hm.get("channel"); if(channel == "") channel = null;
String subject = null;
String imgsrc = hm.get("imgsrc"); if(imgsrc == "") imgsrc = null;
String videosrc = hm.get("videosrc");
if(videosrc == "") videosrc = null;
else {
videosrc = "http://www.reuters.com" + videosrc;
videosrc = videosrc.replaceAll(" ", "");
}
String str = list.get(i)+"\t"+title+"\t"+content+"\t"+time+"\t"+publisher+"\t"
+site+"\t"+reporter+"\t"+channel+"\t"+subject+"\t"+imgsrc+"\t"+videosrc+"\n";

System.out.println("URL： " + list.get(i));
System.out.println("标题： " + title);
System.out.println("正文： " + content);
System.out.println("发布时间： " + time);
System.out.println("发布者：" + publisher);
System.out.println("来源站点：" + site);
System.out.println("记者：" + reporter);
System.out.println("分类频道：" + channel);
System.out.println("主题：" + subject);
System.out.println("图片链接：" + imgsrc);
System.out.println("视频链接：" + videosrc);
System.out.println(str);

try {
FileOutputStream fos = new FileOutputStream("D://News.txt",true);
OutputStreamWriter osw = new OutputStreamWriter(fos, "utf-8");
BufferedWriter bw = new BufferedWriter(osw);

bw.write(str);
bw.flush();
bw.close();
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}
}
}