go连接mysql,redis并完成日志字符处理实例

go连接mysql,redis并完成日志字符处理实例

package main// 引入所需包import (    "compress/gzip"    "fmt"    "io/ioutil"    "os"    "strings"    //"reflect"    "github.com/garyburd/redigo/redis"    "math"    "strconv"    "net/url"    "time"    "database/sql"    _ "github.com/go-sql-driver/mysql")//获取指定目录下的所有gz文件,包含子目录下的gz文件func GetGzFiles(dirPth string) (files []string, err error) {    var dirs []string    dir, err := ioutil.ReadDir(dirPth)    if err != nil {        return nil, err    }    PthSep := string(os.PathSeparator)    //suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写    for _, fi := range dir {        if fi.IsDir() { // 目录, 递归遍历            dirs = append(dirs, dirPth+PthSep+fi.Name())            GetGzFiles(dirPth + PthSep + fi.Name())        } else {            // 过滤指定格式<.gz文件>            ok := strings.HasSuffix(fi.Name(), ".gz")            if ok {                files = append(files, dirPth+PthSep+fi.Name())            }        }    }    // 读取子目录下文件    for _, dir := range dirs {        tmpFiles, _ := GetGzFiles(dir)        for _, file := range tmpFiles {            files = append(files, file)        }    }    return files, nil}/*需要记录的值包括:    访问时间,日期,状态码,cache返回的状态码,缓存状态码,源站返回状态码,节点IP,客户端IP,响应时间,故障原因故障原因包括:    (1)nginx连接cache超时(2)nginx读取cache响应头超时(3)nginx产生的其它异常(4)cache连接源站失败(5)cache缓存住的错误码(6)cache上层返回的错误码(7)cache给出的其它异常(8)其它异常*/type ResInfo struct {    id    int   // 唯一ID    timestamp int64 // 访问时间    date      string// 日期    dnid      int   // 域名ID    resocde   int   // 状态码,206    cacheCode int   // cache返回的状态码,200    upstreamHttpStatus string   // 缓存状态码,HIT    sourceCode int  // 源站返回状态码,200    errorCode int   // 故障原因,0-9    clientIp  string    // 客户端IP    serverIp  string    // 节点IP    reponseTime float64 // 响应时间}var id int = 0func main() {    // 更新域名列表    db, err := sql.Open("mysql", "user:passwd@tcp(127.0.0.1:65000)/server_conf?charset=utf8")    if err != nil {        fmt.Printf("connect mysql failed! [%s]", err)        return    } else {        fmt.Println("connect mysql ok!")    }    rows, err := db.Query("SELECT d_id, d_dname FROM domain")    if err != nil {        fmt.Printf("select failed! [%s]", err)        return    }    var mapDomain map[string]int    mapDomain = make(map[string]int)    for rows.Next() {        var d_id int        var d_dname string        rows.Columns()        err = rows.Scan(&d_id, &d_dname)        if err != nil {            fmt.Printf("Get domain info failed! [%s]", err)        }        mapDomain[d_dname] = d_id    }    //for k, v := range mapDomain {    //  fmt.Println(k, v)    //}    // 读取待处理的文件    files, _ := GetGzFiles("./gzfiles")    var ResInfoList []ResInfo    for _, fn := range files {        fmt.Printf("开始处理日志文件[%s]\n", fn)        // 打开本地gz格式压缩包        fr, err := os.Open(fn)        if err != nil {            panic(err)        } else {            println("open file success!")        }        // defer: 在函数退出时,执行关闭文件        defer fr.Close()        // 创建gzip文件读取对象        gr, err := gzip.NewReader(fr)        if err != nil {            panic(err)        }        // defer: 在函数退出时,执行关闭gzip对象        defer gr.Close()        // 读取gzip对象内容        rBuf, err := ioutil.ReadAll(gr)        if err != nil {            fmt.Println("[read gzip data err]: ", err)        }        // []uint8转string        // fmt.Println(reflect.TypeOf(rBuf))        strBuf := string(rBuf)        bufList := strings.Split(strBuf, "\n")        for _, strLine := range bufList {            if len(strLine) == 0 {                continue            }            var dnid      int            var errorCode int            var timestamp int64            var resocde   int            var cacheCode int            var sourceCode int            var reponseTime float64            var date      string            var clientIp  string            var serverIp  string            var upstreamHttpStatus string            var userDefineList []string            fragList := strings.Split(strLine, "\t")            for i, strFrag := range fragList {                if i == 0 {                    strTemp := strings.Split(strFrag, "/")                    clientIp = strTemp[0]                    fmt.Printf("clientIp=%v\n", clientIp)                } else if i == 1 {                    tmp, _ := strconv.ParseFloat(strFrag, 64)                    timestamp = int64(math.Trunc(tmp))/60*60                    fmt.Printf("timestamp=%v\n", timestamp)                    tm := time.Unix(timestamp, 0)                    date = tm.Format("20060102")                } else if i == 4 {                    resocde, _ = strconv.Atoi(strFrag)                    fmt.Printf("resocde=%v\n", resocde)                } else if i == 5 {                    upstreamHttpStatus = strFrag                    fmt.Printf("upstreamHttpStatus=%v\n", upstreamHttpStatus)                } else if i == 8 {                    strTemp := strFrag                    u, err := url.Parse(strTemp)                    if err != nil {                        panic(err)                    }                    fmt.Printf("url=%v\n", u.Host)                    // 获取url的dnid                    if v, ok := mapDomain[u.Host]; ok {                        dnid = v                    } else {                        fmt.Printf("Domain Not Found!")                    }                } else if i == 13 {                    serverIp = strFrag                    fmt.Printf("serverIp=%v\n", serverIp)                } else if i == 17 {                    reponseTime, _ = strconv.ParseFloat(strFrag, 64)                    fmt.Printf("reponseTime=%v\n", reponseTime)                } else if i == 29 {                    //fmt.Printf("%s \t", strFrag)                    userDefineList = strings.Split(strFrag, "/")                    for i, strDefine := range userDefineList {                        if i == 5 {                            cacheCode, _ = strconv.Atoi(strDefine)                            //fmt.Printf("cacheCode=%v \n", cacheCode)                            //fmt.Println(reflect.TypeOf(cacheCode))                        }                    }                    //fmt.Printf("%v \t %v \n", timestamp, cacheCode)                }            }            errorCode = 9            id++            info := ResInfo{id,timestamp,date,dnid,resocde,cacheCode,upstreamHttpStatus,sourceCode,errorCode,clientIp,serverIp,reponseTime}            //fmt.Printf("%v\n", info)            ResInfoList = append(ResInfoList, info)        }        break    }    // 入redis库    // 连接redis数据库,指定数据库的IP和端口    conn, err := redis.Dial("tcp", "127.0.0.1:65000")    if err != nil {        fmt.Println("Connect to redis error", err)        return    } else {        fmt.Println("Connect to redis ok.")    }    // 函数退出时关闭连接    defer conn.Close()    // redis授权登陆    conn.Do("AUTH", "xxxxxx")    // 遍历列表并写入redis    fmt.Printf("list len=%v\n", len(ResInfoList))    var tm int64 = 0    for _, info := range ResInfoList {        //fmt.Printf("%v \n", info)        tm = info.timestamp        // 执行List插入        _, err = conn.Do("LPUSH", info.timestamp, info)        if err != nil {            fmt.Println("redis push list failed:", err)        } else {            //fmt.Println("redis lpush ok!")        }        break    }    // 读取List    infoList, err := redis.Strings(conn.Do("LRANGE", tm, 0, -1))    if err != nil {        fmt.Println("redis get list failed:", err)    }    for _, info := range infoList {        fmt.Printf("values: %v\n", info)    }}