golang采集,适合golang初学者入门,笔者亦是新手
来源:互联网 发布:第九交响曲 知乎 编辑:程序博客网 时间:2024/05/17 01:42
将易果生鲜网站作为了本次练习的入口,代码拷贝请勿对目标站进行攻击后果自负
package main
import (
"fmt"
"io/ioutil"
"net/http"
"regexp"
)
type MySpider struct {
indexUrl string
}
func (this MySpider) readUrlBody() (string, error) {
resp, err := http.Get(this.indexUrl)
if err != nil {
return "err", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "err", err
}
return string(body), err
}
func (this MySpider) catchCategoryUrl() []string {
body, _ := this.readUrlBody()
rcg := regexp.MustCompile(`class="catalogs-ad">(?sU:.*)<a href="(http://www.yiguo.com/products/.*?_channelhome.html)"`)
urls := rcg.FindAllStringSubmatch(body, -1)
cateUrl := make([]string, len(urls))
for i, u := range urls {
cateUrl[i] = u[1]
}
return cateUrl
}
func (this MySpider) catchProductInfo() string {
body, _ := this.readUrlBody()
rcg := regexp.MustCompile(`<div class="p_info clearfix">(?sU:.*)<div class="p_name"><a href="http://www.yiguo.com/product/(?U:.*).html" target="_blank">(.*?)</a></div>(?sU:.*)<div class="p_price">(?sU:.*)<strong>(.*?)</strong>(?sU:.*)</div>(?sU:.*)</div>`)
result := rcg.FindAllStringSubmatch(body, -1)
for i := range result {
line := result[i]
fmt.Println(line[1], "<<======>>", line[2])
}
return ""
}
func (this MySpider) run() string {
cateUrls := this.catchCategoryUrl()
for _, u := range cateUrls {
this.indexUrl = u
this.catchProductInfo()
break
}
return ""
}
func main() {
ms := new(MySpider)
ms.indexUrl = "http://www.yiguo.com"
ms.run()
}
0 0
- golang采集,适合golang初学者入门,笔者亦是新手
- golang 采集图片
- Golang
- golang
- golang
- golang
- golang
- GOLANG
- golang
- golang
- golang
- golang
- golang
- golang
- Golang
- Golang
- Golang
- Golang
- stun turn ice等穿越NAT方法
- HDU 1710 二叉树的遍历(已知前序和中序求后序)
- 什么是Java的反射机制
- java重double类型不能用来计算问题
- POI,POI 简介,POI 导出
- golang采集,适合golang初学者入门,笔者亦是新手
- HDOJ 2485 Destroying the bus stations DFS+BFS
- CodingNet - Learning - 15
- lua学习(3)______lua中的基本类型
- Java-Gson解析-02 借助Token解析JSONArray
- 第一个出现一次的字符
- Spring入门<一>
- BSON与JSON
- android开发常用工具类网站