简单的网络爬虫（获取页面中的url）

简单的网络爬虫（获取页面中的url）

package text;import java.io.BufferedReader;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.PrintWriter;import java.net.MalformedURLException;import java.net.URL;import java.net.URLConnection;import java.util.HashSet;import java.util.Set;import java.util.regex.Matcher;import java.util.regex.Pattern;public class SpiderNet {public static void main(String[] args) {URL url = null;URLConnection urlConnection = null;PrintWriter out = null;BufferedReader in = null;//编写正则表达式String regex = "http://[\\w+\\.?/?]+\\.[A-Za-z]+";//定义正则表达式Pattern pattern = Pattern.compile(regex);try {//访问链接,这里选择steam官网主页url = new URL("http://store.steampowered.com/");//建立url的连接connectionurlConnection = url.openConnection();//创建新 PrintWriter,路径D:/mzsds_mzm.txtout = new PrintWriter(new FileWriter("D:/mzsds_mzm.txt"),true);//将url地址的响应读到in中in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));String buffer = null;Set<String> set = new HashSet<String>();//读响应数据到.txt文件中while((buffer=in.readLine())!=null){//匹配正则表达式Matcher matcher = pattern.matcher(buffer);if(matcher.find()){//输出匹配内容set.add(matcher.group());}}for (String str : set) {       out.println(str);  }  } catch (MalformedURLException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}}}

很简单，没有筛选结果，包含了其他的链接，不是很完善。

结果输出：

http://cdn.steamstatic.com.8686c.com/steam/apps/668210/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/346110/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/672730/capsule_184x69.jpg
http://www.valvesoftware.com/jobs.html
http://translation.steampowered.com
http://www.steampowered.com
http://www.valvesoftware.com
http://store.akamai.steamstatic.com/public/css/v6/home.css
http://www.w3.org
http://store.akamai.steamstatic.com
http://store.akamai.steamstatic.com/public/shared/css/motiva_sans.css
http://cdn.steamstatic.com.8686c.com/steam/apps/434460/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/566160/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_logo.png
http://www.valvesoftware.com/about.html
http://store.akamai.steamstatic.com/public/css/v6/store.css
http://store.steampowered.com
http://cdn.steamstatic.com.8686c.com/steam/apps/686190/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/668630/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/68ca004cca70f207cddabbb4/spotlight_image_english.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/465240/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/cluster.js
http://store.akamai.steamstatic.com/public/shared/images/header/globalheader_logo.png
http://twitter.com
http://store.akamai.steamstatic.com/public/javascript/home.js
http://cdn.steamstatic.com.8686c.com/steam/apps/619880/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/641990/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/buttons.css
http://cdn.steamstatic.com.8686c.com/steam/apps/680320/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/shared_responsive.css
http://store.akamai.steamstatic.com/public/shared/css/shared_global.css
http://cdn.steamstatic.com.8686c.com/steam/apps/515220/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/544610/capsule_184x69.jpg
http://steamcommunity.com
http://store.akamai.steamstatic.com/public/javascript/main.js
http://store.akamai.steamstatic.com/public/shared/javascript/shared_global.js
http://store.akamai.steamstatic.com/public/images/v5/globalheader_logo.png
http://store.akamai.steamstatic.com/public/shared/javascript/shared_responsive_adapter.js
http://cdn.steamstatic.com.8686c.com/steam/apps/555150/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/javascript/tooltip.js
http://cdn.steamstatic.com.8686c.com/steam/apps/577800/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/650780/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_menu_hamburger.png
http://cdn.steamstatic.com.8686c.com/steam/apps/629960/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/689090/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/373930/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/551770/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/559910/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/614570/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/635320/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/435150/capsule_184x69.jpg
http://www.facebook.com
http://cdn.steamstatic.com.8686c.com/steam/apps/554620/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/logo_valve_footer.png
http://cdn.steamstatic.com.8686c.com/steam/apps/607880/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/dynamicstore.js
http://store.akamai.steamstatic.com/public/images/blank.gif
http://cdn.steamstatic.com.8686c.com/steam/apps/449840/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/7830c4a8f396478f946d265b/spotlight_image_english.jpg
http://store.akamai.steamstatic.com/public/shared/images/trans.gif
http://store.akamai.steamstatic.com/public/shared/images/responsive/share_steam_logo.png
http://www.valvesoftware.com/legal.htm
http://cdn.steamstatic.com.8686c.com/steam/apps/357310/capsule_184x69.jpg