CODIS源码剖析二（codis-proxy功能实现）
来源：互联网 发布：java 二叉树 实现 编辑：程序博客网 时间：2024/05/22 04:57
一、codis-proxy启动http服务接受集群命令
func newApiServer(p *Proxy) http.Handler { m := martini.New() //go-martini web开发框架 m.Use(martini.Recovery()) m.Use(render.Renderer()) m.Use(func(w http.ResponseWriter, req *http.Request, c martini.Context) { path := req.URL.Path if req.Method != "GET" && strings.HasPrefix(path, "/api/") { var remoteAddr = req.RemoteAddr var headerAddr string for _, key := range []string{"X-Real-IP", "X-Forwarded-For"} { if val := req.Header.Get(key); val != "" { headerAddr = val break } } log.Warnf("[%p] API call %s from %s [%s]", p, path, remoteAddr, headerAddr) } c.Next() }) m.Use(gzip.All()) m.Use(func(c martini.Context, w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json; charset=utf-8") }) api := &apiServer{proxy: p} r := martini.NewRouter() r.Get("/", func(r render.Render) { r.Redirect("/proxy") }) r.Any("/debug/**", func(w http.ResponseWriter, req *http.Request) { http.DefaultServeMux.ServeHTTP(w, req) }) r.Group("/proxy", func(r martini.Router) { r.Get("", api.Overview) r.Get("/model", api.Model) r.Get("/stats", api.StatsNoXAuth) r.Get("/slots", api.SlotsNoXAuth) }) r.Group("/api/proxy", func(r martini.Router) { r.Get("/model", api.Model) r.Get("/xping/:xauth", api.XPing) r.Get("/stats/:xauth", api.Stats) r.Get("/stats/:xauth/:flags", api.Stats) r.Get("/slots/:xauth", api.Slots) r.Put("/start/:xauth", api.Start) r.Put("/stats/reset/:xauth", api.ResetStats) r.Put("/forcegc/:xauth", api.ForceGC) r.Put("/shutdown/:xauth", api.Shutdown) r.Put("/loglevel/:xauth/:value", api.LogLevel) r.Put("/fillslots/:xauth", binding.Json([]*models.Slot{}), api.FillSlots) r.Put("/sentinels/:xauth", binding.Json(models.Sentinel{}), api.SetSentinels) r.Put("/sentinels/:xauth/rewatch", api.RewatchSentinels) }) m.MapTo(r, (*martini.Routes)(nil)) m.Action(r.Handle) return m}- apiServer提供http服务，包括服务端和客户端Api，当codis-proxy启动之后启动http服务等待集群配置命令的请求，具体的的Api路径在$GOPATH/src/github.com/CodisLabs/codis/pkg/proxy/proxy_api.go；
二、codis-proxy启动redis协议服务处理redis命令
func NewSession(sock net.Conn, config *Config) *Session { c := redis.NewConn(sock, config.SessionRecvBufsize.AsInt(), config.SessionSendBufsize.AsInt(), ) c.ReaderTimeout = config.SessionRecvTimeout.Duration() c.WriterTimeout = config.SessionSendTimeout.Duration() c.SetKeepAlivePeriod(config.SessionKeepAlivePeriod.Duration()) s := &Session{ Conn: c, config: config, //会话的核心结果 CreateUnix: time.Now().Unix(), } s.stats.opmap = make(map[string]*opStats, 16) log.Infof("session [%p] create: %s", s, s) return s}func (s *Session) Start(d *Router) { s.start.Do(func() { if int(incrSessions()) > s.config.ProxyMaxClients { go func() { s.Conn.Encode(redis.NewErrorf("ERR max number of clients reached"), true) s.CloseWithError(ErrTooManySessions) }() decrSessions() return } if !d.isOnline() { go func() { s.Conn.Encode(redis.NewErrorf("ERR router is not online"), true) s.CloseWithError(ErrRouterNotOnline) }() decrSessions() return } tasks := NewRequestChanBuffer(1024) go func() { s.loopWriter(tasks) //等待处理结果并合并结果 decrSessions() }() go func() { s.loopReader(tasks, d) //指令读取及分发（分发给backend） tasks.Close() }() })}- codis-proxy每接到一个redis请求都会创建一个独立的session来处理，下面给出proxy的实现原理。
- 其中Rc对应s.loopReader负责请求读取及请求分发，Wc相当于s.loopWriter负责等待处理结果并合并结果返回给客户端，下面我们会根据r *Request, d *Router为主线进一步分析。
func (s *Session) loopReader(tasks *RequestChan, d *Router) (err error) { defer func() { s.CloseReaderWithError(err) }() var ( breakOnFailure = s.config.SessionBreakOnFailure maxPipelineLen = s.config.SessionMaxPipeline ) for !s.quit { multi, err := s.Conn.DecodeMultiBulk() if err != nil { return err } s.incrOpTotal() if tasks.Buffered() > maxPipelineLen { return ErrTooManyPipelinedRequests } start := time.Now() s.LastOpUnix = start.Unix() s.Ops++ r := &Request{} r.Multi = multi r.Batch = &sync.WaitGroup{} r.Database = s.database r.UnixNano = start.UnixNano() if err := s.handleRequest(r, d); err != nil { //s.handleRequest是指令分发 r.Resp = redis.NewErrorf("ERR handle request, %s", err) tasks.PushBack(r) if breakOnFailure { return err } } else { tasks.PushBack(r) } } return nil}- loopReader将请求分发给backend处理完之后把带有结果的r存储到task中以供loopWriter后续使用。
func (s *Session) handleRequest(r *Request, d *Router) error { opstr, flag, err := getOpInfo(r.Multi) if err != nil { return err } r.OpStr = opstr r.OpFlag = flag r.Broken = &s.broken if flag.IsNotAllowed() { return fmt.Errorf("command '%s' is not allowed", opstr) } switch opstr { case "QUIT": return s.handleQuit(r) case "AUTH": return s.handleAuth(r) } if !s.authorized { if s.config.ProductAuth != "" { r.Resp = redis.NewErrorf("NOAUTH Authentication required") return nil } s.authorized = true } switch opstr { case "SELECT": return s.handleSelect(r) case "PING": return s.handleRequestPing(r, d) case "INFO": return s.handleRequestInfo(r, d) case "MGET": return s.handleRequestMGet(r, d) case "MSET": return s.handleRequestMSet(r, d) case "DEL": return s.handleRequestDel(r, d) case "EXISTS": return s.handleRequestExists(r, d) case "SLOTSINFO": return s.handleRequestSlotsInfo(r, d) case "SLOTSSCAN": return s.handleRequestSlotsScan(r, d) case "SLOTSMAPPING": return s.handleRequestSlotsMapping(r, d) default: return d.dispatch(r) }}- 该方法比较好理解具体命令的处理，下面以
MGET为例
func (s *Session) handleRequestMGet(r *Request, d *Router) error { var nkeys = len(r.Multi) - 1 switch { case nkeys == 0: r.Resp = redis.NewErrorf("ERR wrong number of arguments for 'MGET' command") return nil case nkeys == 1: return d.dispatch(r) //请求分发 } var sub = r.MakeSubRequest(nkeys) //将请求拆分成子请求 for i := range sub { sub[i].Multi = []*redis.Resp{ r.Multi[0], r.Multi[i+1], } if err := d.dispatch(&sub[i]); err != nil { return err } } r.Coalesce = func() error { //绑定结果合并方法，loopWriter中会使用 var array = make([]*redis.Resp, len(sub)) for i := range sub { if err := sub[i].Err; err != nil { return err } switch resp := sub[i].Resp; { case resp == nil: return ErrRespIsRequired case resp.IsArray() && len(resp.Array) == 1: array[i] = resp.Array[0] default: return fmt.Errorf("bad mget resp: %s array.len = %d", resp.Type, len(resp.Array)) } } r.Resp = redis.NewArray(array) //结果保存在r.Resp，以备loopWriter使用 return nil } return nil}- 子请求拆分及方法绑定先不讨论，我们重点看一下d.dispatch，它是命令分发的重要环节，可以将命令分发到具体的codis-server上：
func (s *Router) dispatch(r *Request) error { hkey := getHashKey(r.Multi, r.OpStr) var id = Hash(hkey) % MaxSlotNum //MaxSlotNum=1024 slot := &s.slots[id] //确定槽位，处理方法直接绑定在槽位里面 return slot.forward(r, hkey)}func (s *Slot) forward(r *Request, hkey []byte) error { return s.method.Forward(s, r, hkey)}func (d *forwardSync) Forward(s *Slot, r *Request, hkey []byte) error { s.lock.RLock() bc, err := d.process(s, r, hkey) //获取处理实际实例BackendConn s.lock.RUnlock() if err != nil { return err } bc.PushBack(r) //将请求放入BackendConn等待处理 return nil}func (d *forwardSync) process(s *Slot, r *Request, hkey []byte) (*BackendConn, error) { if s.backend.bc == nil { log.Debugf("slot-%04d is not ready: hash key = '%s'", s.id, hkey) return nil, ErrSlotIsNotReady } if s.migrate.bc != nil && len(hkey) != 0 { if err := d.slotsmgrt(s, hkey, r.Database, r.Seed16()); err != nil { //获取BackendConn log.Debugf("slot-%04d migrate from = %s to %s failed: hash key = '%s', database = %d, error = %s", s.id, s.migrate.bc.Addr(), s.backend.bc.Addr(), hkey, r.Database, err) return nil, err } } r.Group = &s.refs r.Group.Add(1) return d.forward2(s, r), nil //从三个不同的地方获取BackendConn区别不太明白}func (d *forwardHelper) forward2(s *Slot, r *Request) *BackendConn { var database, seed = r.Database, r.Seed16() if s.migrate.bc == nil && r.IsReadOnly() && len(s.replicaGroups) != 0 { for _, group := range s.replicaGroups { var i = seed for _ = range group { i = (i + 1) % uint(len(group)) if bc := group[i].BackendConn(database, seed, false); bc != nil { return bc } } } } return s.backend.bc.BackendConn(database, seed, true)}- 上述逻辑主要就是先从槽位Slot里获取BackendConn处理实例，然后把请求r *Request放入BackendConn的处理队列input(chan *Request,在for循环中chan为空不会终止循环而是会等待新的请求进入)，那么存在一个问题BackendConn在哪里创建的，经过分析得知BackendConn同样存在loopWriter和loopReader，而他的启动见程序：
func NewBackendConn(addr string, database int, config *Config) *BackendConn { bc := &BackendConn{ addr: addr, config: config, database: database, } bc.input = make(chan *Request, 1024) bc.retry.delay = &DelayExp2{ Min: 50, Max: 5000, Unit: time.Millisecond, } go bc.run() return bc}func (bc *BackendConn) run() { log.Warnf("backend conn [%p] to %s, db-%d start service", bc, bc.addr, bc.database) for round := 0; bc.closed.IsFalse(); round++ { log.Warnf("backend conn [%p] to %s, db-%d round-[%d]", bc, bc.addr, bc.database, round) if err := bc.loopWriter(round); err != nil { //bc.loopWriter相当于Ws bc.delayBeforeRetry() } } log.Warnf("backend conn [%p] to %s, db-%d stop and exit", bc, bc.addr, bc.database)}loopWriter->newBackendReader:func (bc *BackendConn) newBackendReader(round int, config *Config) (*redis.Conn, chan<- *Request, error) { c, err := redis.DialTimeout(bc.addr, time.Second*5, config.BackendRecvBufsize.AsInt(), config.BackendSendBufsize.AsInt()) if err != nil { return nil, nil, err } c.ReaderTimeout = config.BackendRecvTimeout.Duration() c.WriterTimeout = config.BackendSendTimeout.Duration() c.SetKeepAlivePeriod(config.BackendKeepAlivePeriod.Duration()) if err := bc.verifyAuth(c, config.ProductAuth); err != nil { c.Close() return nil, nil, err } if err := bc.selectDatabase(c, bc.database); err != nil { c.Close() return nil, nil, err } tasks := make(chan *Request, config.BackendMaxPipeline) go bc.loopReader(tasks, c, round) //loopReader相当于Rs return c, tasks, nil}- 从上面的代码基本可以看出一个执行链：NewBackendConn->run->loopWriter->newBackendReader->loopReader,不过还有个疑惑，没有看到NewBackendConn的调用处，实际上他的调用源头在$GOPATH/src/github.com/CodisLabs/codis/pkg/proxy/router.go的fillSlot里面：
func (s *Router) fillSlot(m *models.Slot, switched bool, method forwardMethod) { slot := &s.slots[m.Id] slot.blockAndWait() slot.backend.bc.Release() slot.backend.bc = nil slot.backend.id = 0 slot.migrate.bc.Release() slot.migrate.bc = nil slot.migrate.id = 0 for i := range slot.replicaGroups { for _, bc := range slot.replicaGroups[i] { bc.Release() } } slot.replicaGroups = nil slot.switched = switched if addr := m.BackendAddr; len(addr) != 0 { slot.backend.bc = s.pool.primary.Retain(addr) slot.backend.id = m.BackendAddrGroupId } if from := m.MigrateFrom; len(from) != 0 { slot.migrate.bc = s.pool.primary.Retain(from) slot.migrate.id = m.MigrateFromGroupId } if !s.config.BackendPrimaryOnly { for i := range m.ReplicaGroups { var group []*sharedBackendConn for _, addr := range m.ReplicaGroups[i] { group = append(group, s.pool.replica.Retain(addr)) } if len(group) == 0 { continue } slot.replicaGroups = append(slot.replicaGroups, group) } } ...}func (p *sharedBackendConnPool) Retain(addr string) *sharedBackendConn { if bc := p.pool[addr]; bc != nil { return bc.Retain() } else { bc = newSharedBackendConn(addr, p) p.pool[addr] = bc return bc }}func newSharedBackendConn(addr string, pool *sharedBackendConnPool) *sharedBackendConn { host, port, err := net.SplitHostPort(addr) if err != nil { log.ErrorErrorf(err, "split host-port failed, address = %s", addr) } s := &sharedBackendConn{ addr: addr, host: []byte(host), port: []byte(port), } s.owner = pool s.conns = make([][]*BackendConn, pool.config.BackendNumberDatabases) for database := range s.conns { parallel := make([]*BackendConn, pool.parallel) for i := range parallel { parallel[i] = NewBackendConn(addr, database, pool.config) //创建BackendConn } s.conns[database] = parallel } if pool.parallel == 1 { s.single = make([]*BackendConn, len(s.conns)) for database := range s.conns { s.single[database] = s.conns[database][0] } } s.refcnt = 1 return s}- 而fillSlot的调用在集群启动的槽位分配操作或者后续槽位调整操作里面来做，至此我们简单分析一下backend里面的loopWriter和loopReader
func (bc *BackendConn) loopWriter(round int) (err error) { defer func() { for i := len(bc.input); i != 0; i-- { r := <-bc.input bc.setResponse(r, nil, ErrBackendConnReset) } log.WarnErrorf(err, "backend conn [%p] to %s, db-%d writer-[%d] exit", bc, bc.addr, bc.database, round) }() c, tasks, err := bc.newBackendReader(round, bc.config) //新建指令处理backend if err != nil { return err } defer close(tasks) defer bc.state.Set(0) bc.state.Set(stateConnected) bc.retry.fails = 0 bc.retry.delay.Reset() p := c.FlushEncoder() p.MaxInterval = time.Millisecond p.MaxBuffered = math2.MinInt(256, cap(tasks)) for r := range bc.input { //循环将请求放入tasks队列中，bc.input空会阻塞等待 if r.IsReadOnly() && r.IsBroken() { bc.setResponse(r, nil, ErrRequestIsBroken) continue } if err := p.EncodeMultiBulk(r.Multi); err != nil { return bc.setResponse(r, nil, fmt.Errorf("backend conn failure, %s", err)) } if err := p.Flush(len(bc.input) == 0); err != nil { return bc.setResponse(r, nil, fmt.Errorf("backend conn failure, %s", err)) } else { tasks <- r } } return nil}func (bc *BackendConn) newBackendReader(round int, config *Config) (*redis.Conn, chan<- *Request, error) { c, err := redis.DialTimeout(bc.addr, time.Second*5, config.BackendRecvBufsize.AsInt(), config.BackendSendBufsize.AsInt()) //创建于redis的链接，池化（newBackendReader初始化调） if err != nil { return nil, nil, err } c.ReaderTimeout = config.BackendRecvTimeout.Duration() c.WriterTimeout = config.BackendSendTimeout.Duration() c.SetKeepAlivePeriod(config.BackendKeepAlivePeriod.Duration()) if err := bc.verifyAuth(c, config.ProductAuth); err != nil { //权限验证 c.Close() return nil, nil, err } if err := bc.selectDatabase(c, bc.database); err != nil { //执行redis命令，获取结果 c.Close() return nil, nil, err } tasks := make(chan *Request, config.BackendMaxPipeline) go bc.loopReader(tasks, c, round) //结果回写 return c, tasks, nil}func (bc *BackendConn) loopReader(tasks <-chan *Request, c *redis.Conn, round int) (err error) { defer func() { c.Close() for r := range tasks { bc.setResponse(r, nil, ErrBackendConnReset) } log.WarnErrorf(err, "backend conn [%p] to %s, db-%d reader-[%d] exit", bc, bc.addr, bc.database, round) }() for r := range tasks { //相当于while循环 resp, err := c.Decode() //结果解码 if err != nil { return bc.setResponse(r, nil, fmt.Errorf("backend conn failure, %s", err)) } if resp != nil && resp.IsError() { switch { case bytes.HasPrefix(resp.Value, errMasterDown): if bc.state.CompareAndSwap(stateConnected, stateDataStale) { log.Warnf("backend conn [%p] to %s, db-%d state = DataStale, caused by 'MASTERDOWN'", bc, bc.addr, bc.database) } } } bc.setResponse(r, resp, nil) //r.Resp= resp,置结果到r *request } return nil}- 最后我们看看s.loopWriter是如何读取结果并返回给客户端的：
func (s *Session) loopWriter(tasks *RequestChan) (err error) { defer func() { s.CloseWithError(err) tasks.PopFrontAllVoid(func(r *Request) { s.incrOpFails(r, nil) }) s.flushOpStats(true) }() var breakOnFailure = s.config.SessionBreakOnFailure p := s.Conn.FlushEncoder() p.MaxInterval = time.Millisecond p.MaxBuffered = 256 return tasks.PopFrontAll(func(r *Request) error { resp, err := s.handleResponse(r) //获取响应结果 if err != nil { resp = redis.NewErrorf("ERR handle response, %s", err) if breakOnFailure { s.Conn.Encode(resp, true) return s.incrOpFails(r, err) } } if err := p.Encode(resp); err != nil { return s.incrOpFails(r, err) } fflush := tasks.IsEmpty() if err := p.Flush(fflush); err != nil { return s.incrOpFails(r, err) } else { s.incrOpStats(r, resp.Type) } if fflush { s.flushOpStats(false) } return nil })}func (c *RequestChan) PopFrontAll(onRequest func(r *Request) error) error { for { r, ok := c.PopFront() if ok { if err := onRequest(r); err != nil { return err } } else { return nil } }}func (s *Session) handleResponse(r *Request) (*redis.Resp, error) { r.Batch.Wait() if r.Coalesce != nil { if err := r.Coalesce(); err != nil { //r.Coalesce合并backend返回的结果 return nil, err } } if err := r.Err; err != nil { return nil, err } else if r.Resp == nil { return nil, ErrRespIsRequired } return r.Resp, nil}- 函数Coalesce()是前面提到在函数func (s *Session) handleRequestMGet(r *Request, d *Router) error 里面绑定的结果合并函数。至此codis-proxy的主要实现原理基本分析完成
阅读全文
0 0
- CODIS源码剖析二（codis-proxy功能实现）
- 豌豆夹Redis解决方案Codis源码剖析：Proxy代理
- [CODIS原理]: CODIS-PROXY实现原理[2.X]
- codis
- Codis
- codis proxy 配制
- Codis源码解析——proxy的启动
- Codis源码解析——proxy监听redis请求
- Codis源码解析——proxy添加到集群
- codis的proxy层HA
- [CODIS原理]: CODIS-HA实现原理
- 豌豆夹Redis解决方案Codis源码剖析：Dashboard
- JAVA实现读写Codis
- Codis源码解析——codis-server添加到集群
- Codis源码解析——dashboard的启动（1）
- Codis源码解析——dashboard的启动（2）
- Codis源码解析——处理slot操作（1）
- Codis源码解析——处理slot操作（2）
- 利用贪心算法求解tsp问题
- 面试某乎
- Android M(Android 6.0)运行时权限开发详细解析
- 17.05.15小总结
- 索引知识点
- CODIS源码剖析二（codis-proxy功能实现）
- Java使用字节流复制一个文件夹
- java.lang.reflect.Field
- HttpClient实现远程调用
- GET和POST区别
- 【JSP学习笔记(3)】——文件操作
- linux cat
- 常用技巧（一）尺取法
- MYSQL修改root密码的四种方法
