【docker 17 源码分析】 Docker Daemon启动

基础知识

     Daemon通过三种方式监听请求，unix，tcp，fd，默认使用unix domain socket(/var/run/Docker.sock)。对于远程请求，可以开启tcp socket(-H tcp:0.0.0.0:2375)，或者固定IP(-H tcp://192.168.0.1:2375)。可以使用多种配置如下：

       $ sudo dockerd -H unix:///var/run/docker.sock -H tcp://192.168.59.106 -H tcp://10.10.10.2

     docker为docker client 和 docker daemon，client端发送命令，daemon端负责执行client发送过来的命令（获取和存储镜像、管理容器等）。两者可以通过TCP，HTTP和UNIX SOCKET来进行通信

• 创建 Docker 运行环境
• httpserver 服务于Docker Client，接收并处理相应请求

一. Docker Daemon 启动源码分析

     入口代码为cmd/dockerd/docker.go。

func main() {       if reexec.Init() {              return       }       _, stdout, stderr := term.StdStreams()       logrus.SetOutput(stderr)       cmd := newDaemonCommand()       cmd.SetOutput(stdout)       ......}

     1.1 reexec 是 docker 自己实现的一个 package，https://groups.google.com/forum/#!topic/docker-dev/ePLDji_qBvE 给的解释可能是个过期的代码

     1.2 logrus 第三方的 log 模块，initLogging 就是将输出定向到 stderr。

     1.3 newDaemonCommand

func newDaemonCommand() *cobra.Command {       opts := daemonOptions{              daemonConfig: config.New(),              common:       cliflags.NewCommonOptions(),       }       cmd := &cobra.Command{              Use:           "dockerd [OPTIONS]",              Short:         "A self-sufficient runtime for containers.",              SilenceUsage:  true,              SilenceErrors: true,              Args:          cli.NoArgs,              RunE: func(cmd *cobra.Command, args []string) error {                     opts.flags = cmd.Flags()                     return runDaemon(opts)              },       }       ......       return cmd}

1.3.1 dockerd 启动 deamon 实际运行的是 runDeamon 函数，可以使用 docker -D 启动，进入 runDeamon()后，

初始化 daemonCli，执行 start 方法，runDeamon 中除了 api service 还会 d, err := daemon.NewDaemon(cli.Config, registryService,

containerdRemote)，注册 registry，和 contanerd 等。

1.3.2defaultDaemonConfigFile默认配置文件"/etc/docker/daemon.json"

func runDaemon(opts daemonOptions) error {       daemonCli := NewDaemonCli()       ......       err = daemonCli.start(opts)       notifyShutdown(err)       return err}

type DaemonCli struct {       *config.Config       configFile *string       flags      *pflag.FlagSet       api             *apiserver.Server       d               *daemon.Daemon       authzMiddleware *authorization.Middleware}

    1.4  daemonCli.start cli.Pidfile 为 /var/run/docker.pid，创建文件并写入 pid

if cli.Pidfile != "" {       pf, err := pidfile.New(cli.Pidfile)       if err != nil {              return fmt.Errorf("Error starting daemon: %v", err)       }       defer func() {              if err := pf.Remove(); err != nil {                     logrus.Error(err)              }       }()}

     1.5  daemonCli.start 启动一个server

• len(cli.Config.Hosts)，如果没有-H参数，长度为0，默认使用的是 /var/run/docker.sock。make一个长度为1的切片
• api server.New(serverConfig) 生成一个api server 对象，包含多个http服务

func (cli *DaemonCli) start(opts daemonOptions) (err error) {       ......       serverConfig := &apiserver.Config{              Logging:     true,              SocketGroup: cli.Config.SocketGroup,              Version:     dockerversion.Version,              EnableCors:  cli.Config.EnableCors,              CorsHeaders: cli.Config.CorsHeaders,       }       if len(cli.Config.Hosts) == 0 {              cli.Config.Hosts = make([]string, 1)       }       api := apiserver.New(serverConfig)       cli.api = api}

    1.6  daemonCli.start 初始化监听地址，因为启动没有指定任何 host, ParseHost 设置为默认值，unix:///var/run/docker.sock，proto 为 unix，addr 为 /var/run/docker.sock，listeners.Init 创建 socket server，accept 客户端发来的请求。其他的 TCP 协议等一样的分析。

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {       ......       for i := 0; i < len(cli.Config.Hosts); i++ {              var err error              if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {                     return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)              }              protoAddr := cli.Config.Hosts[i]              protoAddrParts := strings.SplitN(protoAddr, "://", 2)              if len(protoAddrParts) != 2 {                     return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)              }              proto := protoAddrParts[0]              addr := protoAddrParts[1]              ......              ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)              if err != nil {                     return err              }              ls = wrapListeners(proto, ls)              ......              hosts = append(hosts, protoAddrParts[1])              cli.api.Accept(addr, ls...)       }

// Accept sets a listener the server accepts connections into.func (s *Server) Accept(addr string, listeners ...net.Listener) {       for _, listener := range listeners {              httpServer := &HTTPServer{                     srv: &http.Server{                            Addr: addr,                     },                     l: listener,              }              s.servers = append(s.servers, httpServer)       }}

    1.7  daemonCli.start 生成一个 DefaultService 结构体：

registryService := registry.NewService(cli.Config.ServiceOptions)

// DefaultService is a registry service. It tracks configuration data such as a list// of mirrors.type DefaultService struct {       config *serviceConfig       mu     sync.Mutex}// NewService returns a new instance of DefaultService ready to be// installed into an engine.func NewService(options ServiceOptions) *DefaultService {       return &DefaultService{              config: newServiceConfig(options),       }}

    1.8  daemonCli.start 生成一个 libcontainerd remote 实例：创建目录 /var/run/docker/libcontainerd，目录下 rpc addr 为 docker-containerd.sock

// New creates a fresh instance of libcontainerd remote.func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {       r := &remote{              stateDir:    stateDir,              daemonPid:   -1,              eventTsPath: filepath.Join(stateDir, eventTimestampFilename),       }       for _, option := range options {              if err := option.Apply(r); err != nil {                     return nil, err              }       }       if err := sysinfo.MkdirAll(stateDir, 0700); err != nil {       if r.rpcAddr == "" {              r.rpcAddr = filepath.Join(stateDir, containerdSockFilename)       }       if r.startDaemon {              if err := r.runContainerdDaemon(); err != nil {       return r, nil}

     1.8.1 runContainerDaemon，pid 文件为 /var/run/docker/libcontainerd/docker-containerd.pid，打开文件查看先前进程是否存在，否则启动一个新的实例，命令大致为：docker-containerd -l unix:///var/run/docker/libcontainerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd --shim docker-containerd-shim --runtime docker-runc

写入 pid 至文件。setOOMScore 写入 /proc/${pid}/oom_score_adj

func (r *remote) runContainerdDaemon() error {       pidFilename := filepath.Join(r.stateDir, containerdPidFilename)       f, err := os.OpenFile(pidFilename, os.O_RDWR|os.O_CREATE, 0600)       if n > 0 {              pid, err := strconv.ParseUint(string(b[:n]), 10, 64)                     if system.IsProcessAlive(int(pid)) {                     logrus.Infof("libcontainerd: previous instance of containerd still alive (%d)", pid)                     r.daemonPid = int(pid)                     return nil              }       }       // Start a new instance       args := []string{              "-l", fmt.Sprintf("unix://%s", r.rpcAddr),              "--metrics-interval=0",              "--start-timeout", "2m",              "--state-dir", filepath.Join(r.stateDir, containerdStateDir),       }       ......       cmd := exec.Command(containerdBinary, args...)       if err := cmd.Start(); err != nil {         r.daemonPid = cmd.Process.Pid       return nil}

    1.8.1.1 执行可以查看 ps axf | grep docker，启动进程 docker-containerd

    1.8.2 启动完 docker-containerd 进程，使用 rpc 连接到这个进程，

func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {       dialOpts := []grpc.DialOption{              grpc.WithInsecure(),              grpc.WithBackoffMaxDelay(2 * time.Second),              grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {                     return net.DialTimeout("unix", addr, timeout)              }),       }       conn, err := grpc.Dial(r.rpcAddr, dialOpts...)       r.rpcConn = conn       r.apiClient = containerd.NewAPIClient(conn)       // Get the timestamp to restore from       t := r.getLastEventTimestamp()       tsp, err := ptypes.TimestampProto(t)            r.restoreFromTimestamp = tsp       go r.handleConnectionChange()       if err := r.startEventsMonitor(); err != nil {       return r, nil}

    handleConnectionChange 启动 for 循环，500 毫秒一次进行检查。

    1.9 daemon.NewDaemon 放入第二章节详解

d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)if err != nil {       return fmt.Errorf("Error starting daemon: %v", err)}

    1.10 initRouter(routerOptions)，路由有 checkoutpoint，container，image，volume 等。POST DELETE GET 操作

func initRouter(opts routerOptions) {       decoder := runconfig.ContainerDecoder{}       routers := []router.Router{              // we need to add the checkpoint router before the container router or the DELETE gets masked              checkpointrouter.NewRouter(opts.daemon, decoder),              container.NewRouter(opts.daemon, decoder),              image.NewRouter(opts.daemon, decoder),              systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),              volume.NewRouter(opts.daemon),              build.NewRouter(opts.buildBackend, opts.daemon),              sessionrouter.NewRouter(opts.sessionManager),              swarmrouter.NewRouter(opts.cluster),              pluginrouter.NewRouter(opts.daemon.PluginManager()),              distributionrouter.NewRouter(opts.daemon),       }       if opts.daemon.NetworkControllerEnabled() {              routers = append(routers, network.NewRouter(opts.daemon, opts.cluster))       }       if opts.daemon.HasExperimental() {              for _, r := range routers {                     for _, route := range r.Routes() {                            if experimental, ok := route.(router.ExperimentalRoute); ok {                                   experimental.Enable()                            }                     }              }       }       opts.api.InitRouter(routers...)}

         1.10.1 initRouter 主要的是 InitRouter 函数实用的是如下：

func (s *Server) InitRouter(routers ...router.Router) {       s.routers = append(s.routers, routers...)       m := s.createMux()       s.routerSwapper = &routerSwapper{              router: m,       }}// createMux initializes the main router the server uses.func (s *Server) createMux() *mux.Router {       m := mux.NewRouter()       for _, apiRouter := range s.routers {              for _, r := range apiRouter.Routes() {                     f := s.makeHTTPHandler(r.Handler())                     m.Path(versionMatcher + r.Path()).Methods(r.Method()).Handler(f)                     m.Path(r.Path()).Methods(r.Method()).Handler(f)              }       }       debugRouter := debug.NewRouter()       s.routers = append(s.routers, debugRouter)       for _, r := range debugRouter.Routes() {              f := s.makeHTTPHandler(r.Handler())              m.Path("/debug" + r.Path()).Handler(f)       }       err := errors.NewRequestNotFoundError(fmt.Errorf("page not found"))       notFoundHandler := httputils.MakeErrorHandler(err)       m.HandleFunc(versionMatcher+"/{path:.*}", notFoundHandler)       m.NotFoundHandler = notFoundHandler       return m}

         1.11 cli.api.Wait 等待请求在 /var/run/docker.sock

func (s *Server) Wait(waitChan chan error) {       if err := s.serveAPI(); err != nil {              logrus.Errorf("ServeAPI error: %v", err)              waitChan <- err              return       }       waitChan <- nil}

docker 启动总结：

    大部分是参数的检查

    启动 API server 监听请求

    建立 API 路由

    启动进程 libcontainerd，

    镜像之间关系等

    

二. NewDaemon 函数源码分析

       位置：daemon/daemon.go

setDefaultMtu(config)

       2.1 设置 MTU。若 config 中有值则使用 config 中的，否则设置为默认的 1500

func verifyDaemonSettings(conf *config.Config) error {       // Check for mutually incompatible config options       if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {              return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")       }       if !conf.BridgeConfig.EnableIPTables && !conf.BridgeConfig.InterContainerCommunication {              return fmt.Errorf("You specified --iptables=false with --icc=false. ICC=false uses iptables to function. Please set --icc or --iptables to true")       }       if !conf.BridgeConfig.EnableIPTables && conf.BridgeConfig.EnableIPMasq {              conf.BridgeConfig.EnableIPMasq = false       }       if err := VerifyCgroupDriver(conf); err != nil {              return err       }       if conf.CgroupParent != "" && UsingSystemd(conf) {              if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") {                     return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")              }       }       if conf.DefaultRuntime == "" {              conf.DefaultRuntime = config.StockRuntimeName       }       if conf.Runtimes == nil {              conf.Runtimes = make(map[string]types.Runtime)       }       conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary}       return nil}

      2.2 验证 daemon 的配置

• 不能同时指定网桥和网桥的 IP。如果指定网桥，应使用网桥的当前IP地址，不能再设置 IP 地址。
• 不能同时禁用 iptable 和 icc(容器间通信)。如果禁用 icc，docker 会在宿主机的 iptables 的 FORWARD chain 中添加一条 docker 容器间流量均 DROP 的规则，此时设置 EnableIPTables 为f alse，冲突!!!

func isBridgeNetworkDisabled(conf *config.Config) bool {       return conf.BridgeConfig.Iface == config.DisableNetworkBridge}

    2.3 设置是否启用网桥。disableNetworkBridge 为 none，Iface 为空，所以 DisableBridge 为false

if !platformSupported {       return nil, errSystemNotSupported}if err := checkSystem(); err != nil {       return nil, err}

   2.4 检查系统支持和用户权限。需要 root 权限，linux 中 uid = 0 为 root 用户，检查内核版本

uidMaps, gidMaps, err := setupRemappedRoot(config)rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)

   2.5 docker 支持 user namespace，主要隔离安全相关 identifiers 和 attributes，包括用户 ID、用户组 ID、root 目录、key 以及特殊权限。普通用户的进程通过 clone() 创建的新进程在新 user namespace 中拥有不同的用户和用户组，这意味着一个进程在容器外属于一个没有特权的普通用户，但是创建的容器进程却属于拥有所有权限的超级用户。

// setupDaemonProcess sets various settings for the daemon's processfunc setupDaemonProcess(config *config.Config) error {       // setup the daemons oom_score_adj       return setupOOMScoreAdj(config.OOMScoreAdjust)}

   2.6 设置OOM killer值。oom killer设置/proc/self/oom_score_adj，值的范围为–17~15。正值易被OOM Killer选定。如果设置为–17，表示禁止被kill掉

tmp, err := prepareTempDir(config.Root, rootIDs)realTmp, err := getRealPath(tmp)os.Setenv("TMPDIR", realTmp)

   2.7  prepareTempDir 设置 tmp dir。如有环境变量 DOCKER_TMPDIR，则使用这个，否则创建 /var/lib/docker/tmp，将旧的变为 /var/lib/docker/tmp-old。getRealPath 然后创建一个指向tmp文件的符号链接realTmp，并把 realTmp 赋值给环境变量TMPDIR

// configureMaxThreads sets the Go runtime max threads threshold// which is 90% of the kernel setting from /proc/sys/kernel/threads-maxfunc configureMaxThreads(config *config.Config) error {       mt, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")       if err != nil {              return err       }       mtint, err := strconv.Atoi(strings.TrimSpace(string(mt)))       if err != nil {              return err       }       maxThreads := (mtint / 100) * 90       debug.SetMaxThreads(maxThreads)       logrus.Debugf("Golang's threads limit set to %d", maxThreads)       return nil}

    2.8  设置最大线程数。从 /proc/sys/kernel/threads-max 文件读取值，然后乘以0.9

daemonRepo := filepath.Join(config.Root, "containers")if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {       return nil, err}

   2.9  创建容器目录，位于 docker daemon 目录下的 containers。daemon 创建容器会把容器的元数据信息放此

driverName := os.Getenv("DOCKER_DRIVER")if driverName == "" {       driverName = config.GraphDriver}

d.stores[runtime.GOOS] = daemonStore{graphDriver: driverName}
d.RegistryService = registryServiced.PluginStore = pluginStore// Plugin system initialization should happen before restore. Do not change order.d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{       Root:               filepath.Join(config.Root, "plugins"),       ExecRoot:           getPluginExecRoot(config.Root),       Store:              d.PluginStore,       Executor:           containerdRemote,       RegistryService:    registryService,       LiveRestoreEnabled: config.LiveRestoreEnabled,       LogPluginEvent:     d.LogPluginEvent, // todo: make private       AuthzMiddleware:    config.AuthzMiddleware,})if err != nil {       return nil, errors.Wrap(err, "couldn't create plugin manager")}d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{       StorePath:                 config.Root,       MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),       GraphDriver:               driverName,       GraphDriverOptions:        config.GraphOptions,       UIDMaps:                   uidMaps,       GIDMaps:                   gidMaps,       PluginGetter:              d.PluginStore,       ExperimentalEnabled:       config.Experimental,})if err != nil {       return nil, err}

     2.10 填充 daemon 结构体

// Plugin system initialization should happen before restore. Do not change order.d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{       Root:               filepath.Join(config.Root, "plugins"),       ExecRoot:           getPluginExecRoot(config.Root),       Store:              d.PluginStore,       Executor:           containerdRemote,       RegistryService:    registryService,       LiveRestoreEnabled: config.LiveRestoreEnabled,       LogPluginEvent:     d.LogPluginEvent, // todo: make private       AuthzMiddleware:    config.AuthzMiddleware,})if err != nil {       return nil, errors.Wrap(err, "couldn't create plugin manager")}

    2.11 结构体如下 2.11.1 所示，Root 路径为 /var/lib/docker/plugins，ExecRoot 为 /run/docker/plugins，NewManager 生成一个plugin manager，如下 2.11.2 所示      

    2.11.1 结构体 ManagerConfig如下所示：

// ManagerConfig defines configuration needed to start new manager.type ManagerConfig struct {       Store              *Store // remove       Executor           libcontainerd.Remote       RegistryService    registry.Service       LiveRestoreEnabled bool // TODO: remove       LogPluginEvent     eventLogger       Root               string       ExecRoot           string       AuthzMiddleware    *authorization.Middleware}

     2.11.2 NewManager 函数主要是生成目录 /var/lib/docker/plugins，/run/docker/plugins，/var/lib/docker/plugins/tmp，/var/lib/docker/plugins/storage/blobs/tmp ，并初始化结构体 Manager，如下所示：

func NewManager(config ManagerConfig) (*Manager, error) {       if config.RegistryService != nil {              config.RegistryService = pluginRegistryService{config.RegistryService}       }       manager := &Manager{              config: config,       }       if err := os.MkdirAll(manager.config.Root, 0700); err != nil {              if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {              if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {                var err error       manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct              manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))             manager.cMap = make(map[*v2.Plugin]*controller)       manager.publisher = pubsub.NewPublisher(0, 0)       return manager, nil}

    

    2.12 如下所示，StoreOptions 结构体如下 2.12.1 所示，NewStoreFromOptions 函数 2.12.2 所示：

for platform, ds := range d.stores {       ls, err := layer.NewStoreFromOptions(layer.StoreOptions{              StorePath:                 config.Root,              MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),              GraphDriver:               ds.graphDriver,              GraphDriverOptions:        config.GraphOptions,              IDMappings:                idMappings,              PluginGetter:              d.PluginStore,              ExperimentalEnabled:       config.Experimental,              Platform:                  platform,       })              ds.graphDriver = ls.DriverName() // As layerstore may set the driver       ds.layerStore = ls       d.stores[platform] = ds       graphDrivers = append(graphDrivers, ls.DriverName())}

    2.12.1 StoreOptions 结构体如下所示：

// StoreOptions are the options used to create a new Store instancetype StoreOptions struct {       StorePath                 string       MetadataStorePathTemplate string       GraphDriver               string       GraphDriverOptions        []string       IDMappings                *idtools.IDMappings       PluginGetter              plugingetter.PluginGetter       ExperimentalEnabled       bool       Platform                  string}

    2.12.2 NewStoreFromOptions 函数中的 driver 为 aufs，创建目录 /var/lib/docker/images/aufs/layerdb，如下所示：

// NewStoreFromOptions creates a new Store instancefunc NewStoreFromOptions(options StoreOptions) (Store, error) {       driver, err := graphdriver.New(options.GraphDriver, options.PluginGetter, graphdriver.Options{              Root:                options.StorePath,              DriverOptions:       options.GraphDriverOptions,              UIDMaps:             options.IDMappings.UIDs(),              GIDMaps:             options.IDMappings.GIDs(),              ExperimentalEnabled: options.ExperimentalEnabled,       })       if err != nil {              return nil, fmt.Errorf("error initializing graphdriver: %v", err)       }       logrus.Debugf("Using graph driver %s", driver)       fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))       if err != nil {              return nil, err       }       return NewStoreFromGraphDriver(fms, driver, options.Platform)}

d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads)d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)

    2.13 创建上传下载管理器。设置了上传/下载的最大并发数

for platform, ds := range d.stores {       imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)       ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))         var is image.Store       is, err = image.NewImageStore(ifs, platform, ds.layerStore)       ds.imageRoot = imageRoot       ds.imageStore = is       d.stores[platform] = ds}

    2.14 创建镜像存储目录并restore镜像。存储镜像的根目录为 /var/lib/docker/image/aufs/imagedb，然后还会在 imagedb 目录下新建两个目录，content/sha256和metadata/sha256，存储镜像内容和镜像元数据

func (daemon *Daemon) configureVolumes(rootUID, rootGID int) (*store.VolumeStore, error) {       volumesDriver, err := local.New(daemon.configStore.Root, rootUID, rootGID)       if err != nil {              return nil, err       }       volumedrivers.RegisterPluginGetter(daemon.PluginStore)       if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) {              return nil, errors.New("local volume driver could not be registered")       }       return store.New(daemon.configStore.Root)}

    2.14 配置 volumes。设置数据卷 driver，数据卷是容器之间进行数据共享的一种手段；数据卷可以是一个本机命令（通过-v 标识挂载在到容器的某个目录下）数据卷也可以作为数据卷容器，通过--volumes-from挂载到某个容器中；

    local.New 创建目录 /var/lib/docker/volumes，容器的 volume 是可从宿主机上挂载到容器内的特定目录。一个 volume 可以被多个容器挂载共享数据。volumes 以插件形式，这里的 local.New 是创建了默认的 volume driver 的实现

trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath)if err != nil {       return nil, err}

     2.15 创建 key，路径在/etc/docker/key.json

trustDir := filepath.Join(config.Root, "trust")if err := system.MkdirAll(trustDir, 0700, ""); err != nil {       return nil, err}

     2.17 创建 trust 目录，路径在/var/lib/docker/trust

// New returns new *Events instancefunc New() *Events {       return &Events{              events: make([]eventtypes.Message, 0, eventsLimit),              pub:    pubsub.NewPublisher(100*time.Millisecond, bufferSize),       }}

     2.18 创建event实例，产生的event通过订阅的方式获取

// NewReferenceStore creates a new reference store, tied to a file path where// the set of references are serialized in JSON format.func NewReferenceStore(jsonPath string) (Store, error) {       abspath, err := filepath.Abs(jsonPath)       if err != nil {              return nil, err       }       store := &store{              jsonPath:            abspath,              Repositories:        make(map[string]repository),              referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),       }       // Load the json file if it exists, otherwise create it.       if err := store.reload(); os.IsNotExist(err) {              if err := store.save(); err != nil {                     return nil, err              }       } else if err != nil {              return nil, err       }       return store, nil}

type store struct {       mu sync.RWMutex       // jsonPath is the path to the file where the serialized tag data is       // stored.       jsonPath string       // Repositories is a map of repositories, indexed by name.       Repositories map[string]repository       // referencesByIDCache is a cache of references indexed by ID, to speed       // up References.       referencesByIDCache map[digest.Digest]map[string]reference.Named}

    2.19  创建TagStore。用于管理存储镜像的仓库列表，Repositories记录了镜像仓库的映射，
referencesByIDCache记录了镜像ID和镜像全名的映射

// initDiscovery initializes the discovery watcher for this daemon.func (daemon *Daemon) initDiscovery(conf *config.Config) error {       advertise, err := config.ParseClusterAdvertiseSettings(conf.ClusterStore, conf.ClusterAdvertise)       if err != nil {              if err == discovery.ErrDiscoveryDisabled {                     return nil              }              return err       }       conf.ClusterAdvertise = advertise       discoveryWatcher, err := discovery.Init(conf.ClusterStore, conf.ClusterAdvertise, conf.ClusterOpts)       if err != nil {              return fmt.Errorf("discovery initialization failed (%v)", err)       }       daemon.discoveryWatcher = discoveryWatcher       return nil}

    2.20  服务发现。docker自带的服务发现，docker集群中使用，维护心跳、服务注册等

sysInfo := sysinfo.New(false)// Check if Devices cgroup is mounted, it is hard requirement for container security,// on Linux.if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled {       return nil, errors.New("Devices cgroup isn't mounted")}

    2.21  获取系统信息。包括cgroup、网络配置(网桥/iptables)和linux安全(AppArmor/Seccomp)等。如下所示：

type SysInfo struct {       // Whether the kernel supports AppArmor or not       AppArmor bool       // Whether the kernel supports Seccomp or not       Seccomp bool       cgroupMemInfo       cgroupCPUInfo       cgroupBlkioInfo       cgroupCpusetInfo       cgroupPids       // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work       IPv4ForwardingDisabled bool       // Whether bridge-nf-call-iptables is supported or not       BridgeNFCallIPTablesDisabled bool       // Whether bridge-nf-call-ip6tables is supported or not       BridgeNFCallIP6TablesDisabled bool       // Whether the cgroup has the mountpoint of "devices" or not       CgroupDevicesEnabled bool}

d.ID = trustKey.PublicKey().KeyID()d.repository = daemonRepod.containers = container.NewMemoryStore()d.execCommands = exec.NewStore()d.referenceStore = referenceStored.distributionMetadataStore = distributionMetadataStored.trustKey = trustKeyd.idIndex = truncindex.NewTruncIndex([]string{})d.statsCollector = d.newStatsCollector(1 * time.Second)d.defaultLogConfig = containertypes.LogConfig{       Type:   config.LogConfig.Type,       Config: config.LogConfig.Config,}d.EventsService = eventsServiced.volumes = volStored.root = config.Rootd.uidMaps = uidMapsd.gidMaps = gidMapsd.seccompEnabled = sysInfo.Seccompd.apparmorEnabled = sysInfo.AppArmord.nameIndex = registrar.NewRegistrar()d.linkIndex = newLinkIndex()d.containerdRemote = containerdRemote

    2.22  填充daemon结构体。

• container.NewMemoryStore() 创建内存store，把container的名字和container的映射存储在map中
• exec.NewStore() 创建commands的store，把commands和config的映射，用于执行容器里面的一些任务
• truncindex.NewTruncIndex([]string{}) 对contaniner或者image进行操作，截取id的一部分即可进行操作
• d.newStatsCollector(1 * time.Second) 1s收集一次运行的容器的状态，包括网络和cgroup状态。

       CPU： /proc/stat

func (daemon *Daemon) restore() error {       containers := make(map[string]*container.Container)       logrus.Info("Loading containers: start.")       dir, err := ioutil.ReadDir(daemon.repository)       if err != nil {              return err       }       for _, v := range dir {              id := v.Name()              container, err := daemon.load(id)

2.24  restore 函数。从 /var/lib/docker/containers 读取所有目录，下面的文件夹的名字就是容器的id，存入 containers map[string]*container.Container 信息并注册如 daemon 结构体中。包括 start remove 操作 containers

         注册容器。注册信息都在内存中，启动daemon时需要重新进行注册

NewDaemon 总结：

      验证参数，image store，containers  等创建，读取 containers目录并重启等，填充 daemon 结构体。