nsq源码分析（3）：nsqd之数据持久化2

当nsqd进程退出时，将内存中的数据写入到磁盘
当nsqd进程启动时，将磁盘中的数据读入到内存

nsqd数据信息结构体

nsqd/nsqd.go

type meta struct {    Topics []struct {        Name     string `json:"name"`   // topic名称        Paused   bool   `json:"paused"` // topic状态        Channels []struct {            Name   string `json:"name"`   // channel名称            Paused bool   `json:"paused"` // channel状态        } `json:"channels"`    } `json:"topics"`}

nsqd退出时写入本地元数据

nsqd/nsqd.go

func (n *NSQD) Exit() {    ...    n.Lock()    err := n.PersistMetadata()    if err != nil {        n.logf("ERROR: failed to persist metadata - %s", err)    }    n.logf("NSQ: closing topics")    for _, topic := range n.topicMap {        topic.Close()    }    n.Unlock()    ...}

nsqd/nsqd.go

// 将topic和channel的信息写入磁盘func (n *NSQD) PersistMetadata() error {    // persist metadata about what topics/channels we have, across restarts    // fileName：nsqd数据文件名。保存topic和channels的相关信息    // 根据用户传参data-path指定数据目录，文件命名为nsqd.dat    fileName := newMetadataFile(n.getOpts())    // old metadata filename with ID, maintained in parallel to enable roll-back    // fileNameID：id json数据文件名称。    // fileNameID文件是fileName文件的软连接，内容和fileName一样，用于回滚操作。文件命名为nsqd.867.dat    // nsqd.867.dat这里的867是opts.ID，根据主机名的哈希值算出的0-1024之间的值    fileNameID := oldMetadataFile(n.getOpts())    n.logf("NSQ: persisting topic/channel metadata to %s", fileName)    // 遍历所有的topic和channel。注：在调用PersistMetadata之前就加上了lock锁    // 拿到所有topic和channel的name和paused字段，paused描述该topic或channel的状态    js := make(map[string]interface{})    topics := []interface{}{}    for _, topic := range n.topicMap {        if topic.ephemeral {            continue        }        topicData := make(map[string]interface{})        topicData["name"] = topic.name        topicData["paused"] = topic.IsPaused()        channels := []interface{}{}        topic.Lock()        for _, channel := range topic.channelMap {            channel.Lock()            if channel.ephemeral {                channel.Unlock()                continue            }            channelData := make(map[string]interface{})            channelData["name"] = channel.name            channelData["paused"] = channel.IsPaused()            channels = append(channels, channelData)            channel.Unlock()        }        topic.Unlock()        topicData["channels"] = channels        topics = append(topics, topicData)    }    js["version"] = version.Binary    js["topics"] = topics    // 序列化成json字符串    data, err := json.Marshal(&js)    if err != nil {        return err    }    // 写入nsqd数据文件（nsqd.dat）    tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())    err = writeSyncFile(tmpFileName, data)    if err != nil {        return err    }    err = os.Rename(tmpFileName, fileName)    if err != nil {        return err    }    // technically should fsync DataPath here    stat, err := os.Lstat(fileNameID)    if err == nil && stat.Mode()&os.ModeSymlink != 0 {        return nil    }    // if no symlink (yet), race condition:    // crash right here may cause next startup to see metadata conflict and abort    // 创建nsqd数据文件的软连接（nsqd.867.dat）    tmpFileNameID := fmt.Sprintf("%s.%d.tmp", fileNameID, rand.Int())    if runtime.GOOS != "windows" {        err = os.Symlink(fileName, tmpFileNameID)    } else {        // on Windows need Administrator privs to Symlink        // instead write copy every time        err = writeSyncFile(tmpFileNameID, data)    }    if err != nil {        return err    }    err = os.Rename(tmpFileNameID, fileNameID)    if err != nil {        return err    }    // technically should fsync DataPath here    return nil}

nsqd启动时加载本地元数据

apps/nsqd/nsqd.go

func (p *program) Start() error {    ...    // 加载元数据    err := nsqd.LoadMetadata()    if err != nil {        log.Fatalf("ERROR: %s", err.Error())    }    ...}

nsqd/nsqd.go

// 加载磁盘中的topic和channel信息func (n *NSQD) LoadMetadata() error {    atomic.StoreInt32(&n.isLoading, 1)    defer atomic.StoreInt32(&n.isLoading, 0)    fn := newMetadataFile(n.getOpts())    // old metadata filename with ID, maintained in parallel to enable roll-back    fnID := oldMetadataFile(n.getOpts())    // 读取nsqd数据文件和nsqd数据文件的软连接文件（用于回滚操作）    data, err := readOrEmpty(fn)    if err != nil {        return err    }    dataID, errID := readOrEmpty(fnID)    if errID != nil {        return errID    }    if data == nil && dataID == nil {        return nil // fresh start    }    if data != nil && dataID != nil {        if bytes.Compare(data, dataID) != 0 {            return fmt.Errorf("metadata in %s and %s do not match (delete one)", fn, fnID)        }    }    if data == nil {        // only old metadata file exists, use it        fn = fnID        data = dataID    }    var m meta    err = json.Unmarshal(data, &m)    if err != nil {        return fmt.Errorf("failed to parse metadata in %s - %s", fn, err)    }    // 将nsqd数据文件的json反序列化到Topic和Channels    for _, t := range m.Topics {        if !protocol.IsValidTopicName(t.Name) {            n.logf("WARNING: skipping creation of invalid topic %s", t.Name)            continue        }        topic := n.GetTopic(t.Name)        if t.Paused {            topic.Pause()        }        for _, c := range t.Channels {            if !protocol.IsValidChannelName(c.Name) {                n.logf("WARNING: skipping creation of invalid channel %s", c.Name)                continue            }            channel := topic.GetChannel(c.Name)            if c.Paused {                channel.Pause()            }        }    }    return nil}