influxdb总结

上一篇转载的文章非常经典，接下来补充几点md.Run -> NewServer -> s.Open() -> s.MetaClient.Open -> s.TSDBStore.Open() -> service.Open -> s.TSDBStore.Open  -> loadShards -> (s *Shard) Open() ->NewEngineshareopen会创建一个很重要的数据结构// NewShardIndex returns a new index for a shard.func NewShardIndex(id uint64, database, path string, opt tsdb.EngineOptions) tsdb.Index {return &ShardIndex{Index: opt.InmemIndex.(*Index),id:    id,opt:   opt,}}注册：tsdb.RegisterIndex(IndexName, func(id uint64, database, path string, opt tsdb.EngineOptions) tsdb.Index {return NewShardIndex(id, database, path, opt)})通过idx, err := NewIndex(s.id, s.database, ipath, s.options)创建和下面的Index区别是这个是单个shard的索引，下面是集合索引每个share会新建一个engineNewServer{//用来记录shard分组信息等，根据时间查询的时候先定位shard(MetaClient.ShardGroupsByTimeRange)MetaClient: meta.NewClient(c.Meta),s.TSDBStore = tsdb.NewStore(c.Data.Dir)}这个结构体下面的Databases包含RetentionPolicies下包含所有shard信息，shard信息包含时间段，所以查询的时候根据database查找shard的时间段筛选// Data represents the top level collection of all metadata.type Data struct {Term      uint64 // associated raft termIndex     uint64 // associated raft indexClusterID uint64Databases []DatabaseInfoUsers     []UserInfo// adminUserExists provides a constant time mechanism for determining// if there is at least one admin user.adminUserExists boolMaxShardGroupID uint64MaxShardID      uint64}还有个比较重要的数据结构,其中的属性series很重要，获取serieskey后可以从中获取series对象type Index struct {mu sync.RWMutexdatabase string// In-memory metadata index, built on load and updated when new series come inmeasurements map[string]*Measurement // measurement name to object and indexseries       map[string]*Series      // map series key to the Series objectlastID       uint64                  // last used series ID. They're in memory only for this shardseriesSketch, seriesTSSketch             *hll.PlusmeasurementsSketch, measurementsTSSketch *hll.Plus}表结构里面存放的是根据tag的key与value的seriesids，通过ids到上面结构查找series对象type Measurement struct {database stringName     string `json:"name,omitempty"`name     []byte // cached version as []bytemu         sync.RWMutexfieldNames map[string]struct{}// in-memory index fieldsseriesByID          map[uint64]*Series              // lookup table for series by their idseriesByTagKeyValue map[string]map[string]SeriesIDs // map from tag key to value to sorted set of series ids// lazyily created sorted series IDssortedSeriesIDs SeriesIDs // sorted list of series IDs in this measurement}---------------------------------------------------------------------------------------------------//遍历data目录，把所有的db文件遍历出来for _, db := range dbDirs {//遍历策略文件夹for _, rp := range rpDirs {//遍历share文件夹// Shard file names are numeric shardIDsfor _, sh := range shardDirs {shard := NewShard(shardID, path, walPath, opt)err = shard.Open()}}}shard.open{//生成索引数据结构idx, err := NewIndex(s.id, s.database, ipath, s.options)// Open engine，生成filestore数据结构，读入wal文件生成cache//(e *Engine) Open()其中会执行 e.FileStore.Open打开tsm文件if err := e.Open(); err != nil {return err}//loads the shard metadata into memoryLoadMetadataIndex(shardID uint64, index tsdb.Index)}eginee.open{//把tsm文件读入后生成index索引文件内存t.accessor = &mmapAccessor{f: f,}index, err := t.accessor.init() -> (m *mmapAccessor) init()}LoadMetadataIndex{//生成series结构//初始化也就是把索引文件读入，这里注意的是索引文件的key有三部分组成，measument + tags + fieldName，所以表是从索引里面获取的InitializeSeries(key, name []byte, tags models.Tags)}// addToIndexFromKey will pull the measurement name, series key, and field name from a composite key and add it to the// database index and measurement fieldsfunc (e *Engine) addToIndexFromKey(key []byte, fieldType influxql.DataType) error {seriesKey, field := SeriesAndFieldFromCompositeKey(key)name := tsdb.MeasurementFromSeriesKey(seriesKey)//专门存放表的字段mf := e.fieldset.CreateFieldsIfNotExists(name)if err := mf.CreateFieldIfNotExists(field, fieldType, false); err != nil {return err}// Build in-memory index, if necessary.if e.index.Type() == inmem.IndexName {tags, _ := models.ParseTags(seriesKey)//生成索引if err := e.index.InitializeSeries(seriesKey, name, tags); err != nil {return err}}return nil}InitializeSeries{    func (i *Index) CreateSeriesIfNotExists//where查询通过此结构获取serieidsm.seriesByTagKeyValue[string(t.Key)] = valueMap}查询:// TagValues returns the tag keys and values in the given database, matching the condition.ExecuteStatement（executeSelectStatement）  ->  createIterators (e.ShardMapper.MapShards -> ShardGroupsByTimeRange(先根据时间判断在哪个shard下))  ->CreateIterator -> createVarRefIterator ((s *Store) TagValues() -> ForEachMeasurementSeriesByExpr)//上面介绍过查询先通过MetaClient的方法获取shardid范围，就能获取index索引得到seriskey集合//获取e.index.TagSets筛选series集合tagSets, err := e.index.TagSets([]byte(measurement), opt)createTagSetIterators -> createTagSetGroupIterators -> createVarRefSeriesIterator -> buildCursor -> buildFloatCursor -> 最终调用KeyCursor ->//根据serieskey后定位相应data数据位置(f *FileStore) locations(key string, t int64, ascending bool)