【docker 17 源码分析】 Docker Daemon启动

基础知识

     Daemon通过三种方式监听请求，unix，tcp，fd，默认使用unix domain socket(/var/run/Docker.sock)。对于远程请求，可以开启tcp socket(-H tcp:0.0.0.0:2375)，或者固定IP(-H tcp://192.168.0.1:2375)。可以使用多种配置如下：

       $ sudo dockerd -H unix:///var/run/docker.sock -H tcp://192.168.59.106 -H tcp://10.10.10.2

     docker为docker client 和 docker daemon，client端发送命令，daemon端负责执行client发送过来的命令（获取和存储镜像、管理容器等）。两者可以通过TCP，HTTP和UNIX SOCKET来进行通信
创建 Docker 运行环境
httpserver 服务于Docker Client，接收并处理相应请求

一. Docker Daemon 启动源码分析

     入口代码为cmd/dockerd/docker.go。

func main() {
if reexec.Init() {
return
}

_, stdout, stderr := term.StdStreams()
logrus.SetOutput(stderr)

cmd := newDaemonCommand()
cmd.SetOutput(stdout)
......
}

     1.1
reexec 是 docker 自己实现的一个 package，https://groups.google.com/forum/#!topic/docker-dev/ePLDji_qBvE
给的解释可能是个过期的代码

 
   1.2 logrus 第三方的 log 模块，initLogging 就是将输出定向到 stderr。

 
   1.3 newDaemonCommand

func newDaemonCommand() *cobra.Command {
opts := daemonOptions{
daemonConfig: config.New(),
common:       cliflags.NewCommonOptions(),
}

cmd := &cobra.Command{
Use:           "dockerd [OPTIONS]",
Short:         "A self-sufficient runtime for containers.",
SilenceUsage:  true,
SilenceErrors: true,
Args:          cli.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
opts.flags = cmd.Flags()
return runDaemon(opts)
},
}
......

return cmd
}

1.3.1 dockerd 启动 deamon 实际运行的是 runDeamon 函数，可以使用 docker -D 启动，进入 runDeamon()后，

初始化
daemonCli，执行 start 方法，runDeamon 中除了 api service 还会 d, err := daemon.NewDaemon(cli.Config, registryService,

containerdRemote)，注册
registry，和 contanerd 等。

1.3.2defaultDaemonConfigFile默认配置文件"/etc/docker/daemon.json"

funcrunDaemon(opts daemonOptions) error {

daemonCli := NewDaemonCli()

......

err = daemonCli.start(opts)
notifyShutdown(err)
return err
}

type DaemonCli struct {
*config.Config
configFile *string
flags      *pflag.FlagSet

api             *apiserver.Server
d               *daemon.Daemon
authzMiddleware *authorization.Middleware
}

    1.4  daemonCli.start cli.Pidfile 为 /var/run/docker.pid，创建文件并写入
pid

if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}
defer func() {
if err := pf.Remove(); err != nil {
logrus.Error(err)
}
}()
}

 
   1.5  daemonCli.start 启动一个server

len(cli.Config.Hosts)，如果没有-H参数，长度为0，默认使用的是 /var/run/docker.sock。make一个长度为1的切片
api server.New(serverConfig) 生成一个api server 对象，包含多个http服务

func (cli *DaemonCli) start(opts daemonOptions) (err error) {
......

serverConfig := &apiserver.Config{
Logging:     true,
SocketGroup: cli.Config.SocketGroup,
Version:     dockerversion.Version,
EnableCors:  cli.Config.EnableCors,
CorsHeaders: cli.Config.CorsHeaders,
}

if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}

api := apiserver.New(serverConfig)
cli.api = api

}

 
  1.6
 daemonCli.start 初始化监听地址，因为启动没有指定任何 host, ParseHost 设置为默认值，unix:///var/run/docker.sock，proto 为 unix，addr 为 /var/run/docker.sock，listeners.Init 创建 socket server，accept 客户端发来的请求。其他的 TCP 协议等一样的分析。

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
......

for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
}

protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
}

proto := protoAddrParts[0]
addr := protoAddrParts[1]
......
ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
return err
}
ls = wrapListeners(proto, ls)
......
hosts = append(hosts, protoAddrParts[1])
cli.api.Accept(addr, ls...)
}

// Accept sets a listener the server accepts connections into.
func (s *Server) Accept(addr string, listeners ...net.Listener) {
for _, listener := range listeners {
httpServer := &HTTPServer{
srv: &http.Server{
Addr: addr,
},
l: listener,
}
s.servers = append(s.servers, httpServer)
}
}

 
  1.7  daemonCli.start 生成一个 DefaultService 结构体：

registryService := registry.NewService(cli.Config.ServiceOptions)

// DefaultService is a registry service. It tracks configuration data such as a list
// of mirrors.
type DefaultService struct {
config *serviceConfig
mu     sync.Mutex
}

// NewService returns a new instance of DefaultService ready to be
// installed into an engine.
func NewService(options ServiceOptions) *DefaultService {
return &DefaultService{
config: newServiceConfig(options),
}
}

    1.8  daemonCli.start 生成一个 libcontainerd remote 实例：创建目录
/var/run/docker/libcontainerd，目录下 rpc addr 为 docker-containerd.sock

// New creates a fresh instance of libcontainerd remote.
func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
r := &remote{
stateDir:    stateDir,
daemonPid:   -1,
eventTsPath: filepath.Join(stateDir, eventTimestampFilename),
}
for _, option := range options {
if err := option.Apply(r); err != nil {
return nil, err
}
}

if err := sysinfo.MkdirAll(stateDir, 0700); err != nil {

if r.rpcAddr == "" {
r.rpcAddr = filepath.Join(stateDir, containerdSockFilename)
}

if r.startDaemon {
if err := r.runContainerdDaemon(); err != nil {

return r, nil
}

     1.8.1 runContainerDaemon，pid
文件为 /var/run/docker/libcontainerd/docker-containerd.pid，打开文件查看先前进程是否存在，否则启动一个新的实例，命令大致为：docker-containerd -l unix:///var/run/docker/libcontainerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd
--shim docker-containerd-shim --runtime docker-runc
写入 pid 至文件。setOOMScore 写入 /proc/${pid}/oom_score_adj

func (r *remote) runContainerdDaemon() error {
pidFilename := filepath.Join(r.stateDir, containerdPidFilename)
f, err := os.OpenFile(pidFilename, os.O_RDWR|os.O_CREATE, 0600)

if n > 0 {
pid, err := strconv.ParseUint(string(b[:n]), 10, 64)

if system.IsProcessAlive(int(pid)) {
logrus.Infof("libcontainerd: previous instance of containerd still alive (%d)", pid)
r.daemonPid = int(pid)
return nil
}
}

// Start a new instance
args := []string{
"-l", fmt.Sprintf("unix://%s", r.rpcAddr),
"--metrics-interval=0",
"--start-timeout", "2m",
"--state-dir", filepath.Join(r.stateDir, containerdStateDir),
}
......

cmd := exec.Command(containerdBinary, args...)
if err := cmd.Start(); err != nil {

r.daemonPid = cmd.Process.Pid
return nil
}

    1.8.1.1 执行可以查看 ps axf | grep docker，启动进程 docker-containerd



    1.8.2 启动完 docker-containerd 进程，使用 rpc 连接到这个进程，

func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
dialOpts := []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithBackoffMaxDelay(2 * time.Second),
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", addr, timeout)
}),
}
conn, err := grpc.Dial(r.rpcAddr, dialOpts...)

r.rpcConn = conn
r.apiClient = containerd.NewAPIClient(conn)

// Get the timestamp to restore from
t := r.getLastEventTimestamp()
tsp, err := ptypes.TimestampProto(t)

r.restoreFromTimestamp = tsp

go r.handleConnectionChange()

if err := r.startEventsMonitor(); err != nil {

return r, nil
}

    handleConnectionChange 启动 for 循环，500 毫秒一次进行检查。

    1.9 daemon.NewDaemon 放入第二章节详解

d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}

    1.10 initRouter(routerOptions)，路由有 checkoutpoint，container，image，volume 等。POST DELETE GET 操作

func initRouter(opts routerOptions) {
decoder := runconfig.ContainerDecoder{}

routers := []router.Router{
// we need to add the checkpoint router before the container router or the DELETE gets masked
checkpointrouter.NewRouter(opts.daemon, decoder),
container.NewRouter(opts.daemon, decoder),
image.NewRouter(opts.daemon, decoder),
systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),
volume.NewRouter(opts.daemon),
build.NewRouter(opts.buildBackend, opts.daemon),
sessionrouter.NewRouter(opts.sessionManager),
swarmrouter.NewRouter(opts.cluster),
pluginrouter.NewRouter(opts.daemon.PluginManager()),
distributionrouter.NewRouter(opts.daemon),
}

if opts.daemon.NetworkControllerEnabled() {
routers = append(routers, network.NewRouter(opts.daemon, opts.cluster))
}

if opts.daemon.HasExperimental() {
for _, r := range routers {
for _, route := range r.Routes() {
if experimental, ok := route.(router.ExperimentalRoute); ok {
experimental.Enable()
}
}
}
}

opts.api.InitRouter(routers...)
}

         1.10.1 initRouter 主要的是 InitRouter 函数实用的是如下：

func (s *Server) InitRouter(routers ...router.Router) {
s.routers = append(s.routers, routers...)

m := s.createMux()
s.routerSwapper = &routerSwapper{
router: m,
}
}

// createMux initializes the main router the server uses.
func (s *Server) createMux() *mux.Router {
m := mux.NewRouter()

for _, apiRouter := range s.routers {
for _, r := range apiRouter.Routes() {
f := s.makeHTTPHandler(r.Handler())

m.Path(versionMatcher + r.Path()).Methods(r.Method()).Handler(f)
m.Path(r.Path()).Methods(r.Method()).Handler(f)
}
}

debugRouter := debug.NewRouter()
s.routers = append(s.routers, debugRouter)
for _, r := range debugRouter.Routes() {
f := s.makeHTTPHandler(r.Handler())
m.Path("/debug" + r.Path()).Handler(f)
}

err := errors.NewRequestNotFoundError(fmt.Errorf("page not found"))
notFoundHandler := httputils.MakeErrorHandler(err)
m.HandleFunc(versionMatcher+"/{path:.*}", notFoundHandler)
m.NotFoundHandler = notFoundHandler

return m
}

         1.11 cli.api.Wait 等待请求在 /var/run/docker.sock

func(s *Server) Wait(waitChan chan error) {
if err := s.serveAPI(); err != nil {
logrus.Errorf("ServeAPI error: %v", err)
waitChan <- err
return}
waitChan <- nil
}

docker 启动总结：

    大部分是参数的检查

    启动 API server 监听请求

    建立 API 路由

    启动进程 libcontainerd，

    镜像之间关系等

    

二. NewDaemon 函数源码分析

       位置：daemon/daemon.go

setDefaultMtu(config)

       2.1 设置
MTU。若 config 中有值则使用 config 中的，否则设置为默认的 1500

funcverifyDaemonSettings(conf *config.Config) error {
// Check for mutually incompatible config options
if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {
return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")
}
if !conf.BridgeConfig.EnableIPTables && !conf.BridgeConfig.InterContainerCommunication {
return fmt.Errorf("You specified --iptables=false with --icc=false. ICC=false uses iptables to function. Please set --icc or --iptables to true")
}
if !conf.BridgeConfig.EnableIPTables && conf.BridgeConfig.EnableIPMasq {
conf.BridgeConfig.EnableIPMasq = false
}
if err := VerifyCgroupDriver(conf); err != nil {
return err
}
if conf.CgroupParent != "" && UsingSystemd(conf) {
if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") {
return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
}
}

if conf.DefaultRuntime == "" {
conf.DefaultRuntime = config.StockRuntimeName
}
if conf.Runtimes == nil {
conf.Runtimes = make(map[string]types.Runtime)
}
conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary}

return nil
}

      2.2 验证 daemon 的配置

不能同时指定网桥和网桥的 IP。如果指定网桥，应使用网桥的当前IP地址，不能再设置 IP 地址。
不能同时禁用 iptable 和 icc(容器间通信)。如果禁用 icc，docker 会在宿主机的 iptables 的 FORWARD chain 中添加一条 docker 容器间流量均 DROP 的规则，此时设置 EnableIPTables 为f alse，冲突!!!

funcisBridgeNetworkDisabled(conf *config.Config) bool {
return conf.BridgeConfig.Iface == config.DisableNetworkBridge
}

    2.3 设置是否启用网桥。disableNetworkBridge 为 none，Iface 为空，所以 DisableBridge 为false

if !platformSupported {
return nil, errSystemNotSupported
}

if err := checkSystem(); err != nil {
return nil, err
}

   2.4
检查系统支持和用户权限。需要 root 权限，linux 中 uid = 0 为 root 用户，检查内核版本

uidMaps, gidMaps, err := setupRemappedRoot(config)

rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)

   2.5 docker 支持 user namespace，主要隔离安全相关 identifiers 和 attributes，包括用户 ID、用户组 ID、root 目录、key 以及特殊权限。普通用户的进程通过 clone() 创建的新进程在新 user namespace 中拥有不同的用户和用户组，这意味着一个进程在容器外属于一个没有特权的普通用户，但是创建的容器进程却属于拥有所有权限的超级用户。

// setupDaemonProcess sets various settings for the daemon's process
func setupDaemonProcess(config *config.Config) error {
// setup the daemons oom_score_adj
return setupOOMScoreAdj(config.OOMScoreAdjust)
}

   2.6 设置OOM killer值。oom killer设置/proc/self/oom_score_adj，值的范围为–17~15。正值易被OOM Killer选定。如果设置为–17，表示禁止被kill掉

tmp, err := prepareTempDir(config.Root, rootIDs)

realTmp, err := getRealPath(tmp)

os.Setenv("TMPDIR", realTmp)

   2.7  prepareTempDir 设置 tmp dir。如有环境变量 DOCKER_TMPDIR，则使用这个，否则创建 /var/lib/docker/tmp，将旧的变为 /var/lib/docker/tmp-old。getRealPath 然后创建一个指向tmp文件的符号链接realTmp，并把 realTmp 赋值给环境变量TMPDIR

// configureMaxThreads sets the Go runtime max threads threshold
// which is 90% of the kernel setting from /proc/sys/kernel/threads-max
func configureMaxThreads(config *config.Config) error {
mt, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")
if err != nil {
return err
}
mtint, err := strconv.Atoi(strings.TrimSpace(string(mt)))
if err != nil {
return err
}
maxThreads := (mtint / 100) * 90
debug.SetMaxThreads(maxThreads)
logrus.Debugf("Golang's threads limit set to %d", maxThreads)
return nil
}

    2.8  设置最大线程数。从 /proc/sys/kernel/threads-max 文件读取值，然后乘以0.9

daemonRepo := filepath.Join(config.Root, "containers")
if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
return nil, err
}

   2.9  创建容器目录，位于 docker daemon 目录下的 containers。daemon 创建容器会把容器的元数据信息放此

driverName := os.Getenv("DOCKER_DRIVER")
if driverName == "" {
driverName = config.GraphDriver
}

d.stores[runtime.GOOS] = daemonStore{graphDriver: driverName}

d.RegistryService = registryServiced.PluginStore = pluginStore// Plugin system initialization should happen before restore. Do not change order.d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{ Root: filepath.Join(config.Root, "plugins"), ExecRoot: getPluginExecRoot(config.Root), Store: d.PluginStore, Executor: containerdRemote, RegistryService: registryService, LiveRestoreEnabled: config.LiveRestoreEnabled, LogPluginEvent: d.LogPluginEvent, // todo: make private AuthzMiddleware: config.AuthzMiddleware,})if err != nil { return nil, errors.Wrap(err, "couldn't create plugin manager")}d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{ StorePath: config.Root, MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), GraphDriver: driverName, GraphDriverOptions: config.GraphOptions, UIDMaps: uidMaps, GIDMaps: gidMaps, PluginGetter: d.PluginStore, ExperimentalEnabled: config.Experimental,})if err != nil { return nil, err}
     2.10 填充 daemon 结构体

// Plugin system initialization should happen before restore. Do not change order.
d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
Root:               filepath.Join(config.Root, "plugins"),
ExecRoot:           getPluginExecRoot(config.Root),
Store:              d.PluginStore,
Executor:           containerdRemote,
RegistryService:    registryService,
LiveRestoreEnabled: config.LiveRestoreEnabled,
LogPluginEvent:     d.LogPluginEvent, // todo: make private
AuthzMiddleware:    config.AuthzMiddleware,
})
if err != nil {
return nil, errors.Wrap(err, "couldn't create plugin manager")
}

    2.11 结构体如下 2.11.1 所示，Root 路径为 /var/lib/docker/plugins，ExecRoot 为 /run/docker/plugins，NewManager 生成一个plugin manager，如下 2.11.2 所示      

    2.11.1 结构体 ManagerConfig如下所示：

// ManagerConfig defines configuration needed to start new manager.
type ManagerConfig struct {
Store              *Store // remove
Executor           libcontainerd.Remote
RegistryService    registry.Service
LiveRestoreEnabled bool // TODO: remove
LogPluginEvent     eventLogger
Root               string
ExecRoot           string
AuthzMiddleware    *authorization.Middleware
}

     2.11.2 NewManager 函数主要是生成目录 /var/lib/docker/plugins，/run/docker/plugins，/var/lib/docker/plugins/tmp，/var/lib/docker/plugins/storage/blobs/tmp ，并初始化结构体 Manager，如下所示：

func NewManager(config ManagerConfig) (*Manager, error) {
if config.RegistryService != nil {
config.RegistryService = pluginRegistryService{config.RegistryService}
}
manager := &Manager{
config: config,
}
if err := os.MkdirAll(manager.config.Root, 0700); err != nil {

if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {

if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {

var err error
manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct

manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))

manager.cMap = make(map[*v2.Plugin]*controller)

manager.publisher = pubsub.NewPublisher(0, 0)
return manager, nil
}

    

    2.12 如下所示，StoreOptions 结构体如下 2.12.1 所示，NewStoreFromOptions 函数 2.12.2 所示：

for platform, ds := range d.stores {
ls, err := layer.NewStoreFromOptions(layer.StoreOptions{
StorePath:                 config.Root,
MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
GraphDriver:               ds.graphDriver,
GraphDriverOptions:        config.GraphOptions,
IDMappings:                idMappings,
PluginGetter:              d.PluginStore,
ExperimentalEnabled:       config.Experimental,
Platform:                  platform,
})

ds.graphDriver = ls.DriverName() // As layerstore may set the driver
ds.layerStore = ls
d.stores[platform] = ds
graphDrivers = append(graphDrivers, ls.DriverName())
}

    2.12.1 StoreOptions 结构体如下所示：

// StoreOptions are the options used to create a new Store instance
type StoreOptions struct {
StorePath                 string
MetadataStorePathTemplate string
GraphDriver               string
GraphDriverOptions        []string
IDMappings                *idtools.IDMappings
PluginGetter              plugingetter.PluginGetter
ExperimentalEnabled       bool
Platform                  string
}

    2.12.2 NewStoreFromOptions 函数中的 driver 为 aufs，创建目录 /var/lib/docker/images/aufs/layerdb，如下所示：

// NewStoreFromOptions creates a new Store instance
func NewStoreFromOptions(options StoreOptions) (Store, error) {
driver, err := graphdriver.New(options.GraphDriver, options.PluginGetter, graphdriver.Options{
Root:                options.StorePath,
DriverOptions:       options.GraphDriverOptions,
UIDMaps:             options.IDMappings.UIDs(),
GIDMaps:             options.IDMappings.GIDs(),
ExperimentalEnabled: options.ExperimentalEnabled,
})
if err != nil {
return nil, fmt.Errorf("error initializing graphdriver: %v", err)
}
logrus.Debugf("Using graph driver %s", driver)

fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))
if err != nil {
return nil, err
}

return NewStoreFromGraphDriver(fms, driver, options.Platform)
}

d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads)

d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)

 
  2.13 创建上传下载管理器。设置了上传/下载的最大并发数

forplatform, ds := range d.stores {
imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)
ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))

var is image.Store
is, err = image.NewImageStore(ifs, platform, ds.layerStore)

ds.imageRoot = imageRoot
ds.imageStore = is
d.stores[platform] = ds
}

    2.14 创建镜像存储目录并restore镜像。存储镜像的根目录为 /var/lib/docker/image/aufs/imagedb，然后还会在 imagedb 目录下新建两个目录，content/sha256和metadata/sha256，存储镜像内容和镜像元数据

func (daemon *Daemon) configureVolumes(rootUID, rootGID int) (*store.VolumeStore, error) {
volumesDriver, err := local.New(daemon.configStore.Root, rootUID, rootGID)
if err != nil {
return nil, err
}

volumedrivers.RegisterPluginGetter(daemon.PluginStore)

if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) {
return nil, errors.New("local volume driver could not be registered")
}
return store.New(daemon.configStore.Root)
}

    2.14 配置 volumes。设置数据卷 driver，数据卷是容器之间进行数据共享的一种手段；数据卷可以是一个本机命令（通过-v 标识挂载在到容器的某个目录下）数据卷也可以作为数据卷容器，通过--volumes-from挂载到某个容器中；

    local.New 创建目录 /var/lib/docker/volumes，容器的 volume 是可从宿主机上挂载到容器内的特定目录。一个 volume 可以被多个容器挂载共享数据。volumes 以插件形式，这里的
local.New 是创建了默认的 volume driver 的实现

trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath)
if err != nil {
return nil, err
}

     2.15 创建 key，路径在/etc/docker/key.json

trustDir := filepath.Join(config.Root, "trust")

if err := system.MkdirAll(trustDir, 0700, ""); err != nil {
return nil, err
}

     2.17 创建 trust 目录，路径在/var/lib/docker/trust

// New returns new *Events instance
func New() *Events {
return &Events{
events: make([]eventtypes.Message, 0, eventsLimit),
pub:    pubsub.NewPublisher(100*time.Millisecond, bufferSize),
}
}

     2.18
创建event实例，产生的event通过订阅的方式获取

// NewReferenceStore creates a new reference store, tied to a file path where
// the set of references are serialized in JSON format.
func NewReferenceStore(jsonPath string) (Store, error) {
abspath, err := filepath.Abs(jsonPath)
if err != nil {
return nil, err
}

store := &store{
jsonPath:            abspath,
Repositories:        make(map[string]repository),
referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {
if err := store.save(); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return store, nil
}

type store struct {
mu sync.RWMutex
// jsonPath is the path to the file where the serialized tag data is
// stored.
jsonPath string
// Repositories is a map of repositories, indexed by name.
Repositories map[string]repository
// referencesByIDCache is a cache of references indexed by ID, to speed
// up References.
referencesByIDCache map[digest.Digest]map[string]reference.Named
}

    2.19  创建TagStore。用于管理存储镜像的仓库列表，Repositories记录了镜像仓库的映射，
referencesByIDCache记录了镜像ID和镜像全名的映射

// initDiscovery initializes the discovery watcher for this daemon.
func (daemon *Daemon) initDiscovery(conf *config.Config) error {
advertise, err := config.ParseClusterAdvertiseSettings(conf.ClusterStore, conf.ClusterAdvertise)
if err != nil {
if err == discovery.ErrDiscoveryDisabled {
return nil
}
return err
}

conf.ClusterAdvertise = advertise
discoveryWatcher, err := discovery.Init(conf.ClusterStore, conf.ClusterAdvertise, conf.ClusterOpts)
if err != nil {
return fmt.Errorf("discovery initialization failed (%v)", err)
}

daemon.discoveryWatcher = discoveryWatcher
return nil
}

    2.20  服务发现。docker自带的服务发现，docker集群中使用，维护心跳、服务注册等

sysInfo := sysinfo.New(false)
// Check if Devices cgroup is mounted, it is hard requirement for container security,
// on Linux.
if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled {
return nil, errors.New("Devices cgroup isn't mounted")
}

    2.21  获取系统信息。包括cgroup、网络配置(网桥/iptables)和linux安全(AppArmor/Seccomp)等。如下所示：

type SysInfo struct {
// Whether the kernel supports AppArmor or not
AppArmor bool
// Whether the kernel supports Seccomp or not
Seccomp bool

cgroupMemInfo
cgroupCPUInfo
cgroupBlkioInfo
cgroupCpusetInfo
cgroupPids

// Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work
IPv4ForwardingDisabled bool

// Whether bridge-nf-call-iptables is supported or not
BridgeNFCallIPTablesDisabled bool

// Whether bridge-nf-call-ip6tables is supported or not
BridgeNFCallIP6TablesDisabled bool

// Whether the cgroup has the mountpoint of "devices" or not
CgroupDevicesEnabled bool
}

d.ID = trustKey.PublicKey().KeyID()
d.repository = daemonRepo
d.containers = container.NewMemoryStore()
d.execCommands = exec.NewStore()
d.referenceStore = referenceStore
d.distributionMetadataStore = distributionMetadataStore
d.trustKey = trustKey
d.idIndex = truncindex.NewTruncIndex([]string{})
d.statsCollector = d.newStatsCollector(1 * time.Second)
d.defaultLogConfig = containertypes.LogConfig{
Type:   config.LogConfig.Type,
Config: config.LogConfig.Config,
}
d.EventsService = eventsService
d.volumes = volStore
d.root = config.Root
d.uidMaps = uidMaps
d.gidMaps = gidMaps
d.seccompEnabled = sysInfo.Seccomp
d.apparmorEnabled = sysInfo.AppArmor

d.nameIndex = registrar.NewRegistrar()
d.linkIndex = newLinkIndex()
d.containerdRemote = containerdRemote

    2.22  填充daemon结构体。

container.NewMemoryStore() 创建内存store，把container的名字和container的映射存储在map中
exec.NewStore() 创建commands的store，把commands和config的映射，用于执行容器里面的一些任务
truncindex.NewTruncIndex([]string{}) 对contaniner或者image进行操作，截取id的一部分即可进行操作
d.newStatsCollector(1 * time.Second) 1s收集一次运行的容器的状态，包括网络和cgroup状态。
       CPU： /proc/stat

func (daemon *Daemon) restore() error {
containers := make(map[string]*container.Container)

logrus.Info("Loading containers: start.")

dir, err := ioutil.ReadDir(daemon.repository)
if err != nil {
return err
}

for _, v := range dir {
id := v.Name()
container, err := daemon.load(id)

2.24  restore 函数。从 /var/lib/docker/containers 读取所有目录，下面的文件夹的名字就是容器的id，存入
containers map[string]*container.Container 信息并注册如 daemon 结构体中。包括 start remove 操作 containers

         注册容器。注册信息都在内存中，启动daemon时需要重新进行注册

NewDaemon 总结：

      验证参数，image store，containers  等创建，读取 containers目录并重启等，填充 daemon 结构体。