LXC1.0.7-- lxc-start 源码分析 01

最近较关心LinuxContainer 的启动流程，所以就从lxc_start.c这个文件看起。

首先进入源文件，直接到main程序来，本人喜欢按照程序执行的顺序来看代码，所以看个人喜好了。

int main(int argc, char *argv[]){    int err = 1;    struct lxc_conf *conf;    //初始化config结构    char *const *args; //传递的参数    char *rcfile = NULL; //指定配置文件    char *const default_args[] = {    //默认的args参数        "/sbin/init",        NULL,    };  struct lxc_container *c;    //lxc-container 的结构体….}

看下lxc_conf这个数据结构

struct lxc_conf {    int is_execute;//容器是否在执行    char *fstab;//fstab？    int tty;//tty的个数    int pts;//pts的个数？    int reboot;//重启？    int need_utmp_watch;//字面翻译 需要utmp 查看    signed long personality;//字面翻译 特点    struct utsname *utsname;//ustname    struct lxc_list cgroup;//cgroup list lxc_list只是简单的链表结构    struct lxc_list id_map;//id_map list    struct lxc_list network;//network list    struct saved_nic *saved_nics;//saved_nics 结构    int num_savednics;//savednics数量？    int auto_mounts;//auto_mounts?    struct lxc_list mount_list;//mount_list list?    struct lxc_list caps;//caps list？    struct lxc_list keepcaps;//keepcaps list?    struct lxc_tty_info tty_info;//tty的相关信息    struct lxc_console console;//console的结构体    struct lxc_rootfs rootfs;//rootfs的结构体    char *ttydir;//tty目录int close_all_fds;//关闭所有fdstruct lxc_list hooks[NUM_LXC_HOOKS];//hooks 函数    char *lsm_aa_profile;   //?    char *lsm_se_context;//?    int tmp_umount_proc;//?    char *seccomp;  // filename with the seccomp rules#if HAVE_SCMP_FILTER_CTX    scmp_filter_ctx seccomp_ctx;#endif    int maincmd_fd;//?    int autodev;  // if 1, mount and fill a /dev at start    int haltsignal; // signal used to halt container    int stopsignal; // signal used to hard stop container    int kmsg;  // if 1, create /dev/kmsg symlink    char *rcfile;   // Copy of the top level rcfile we read    // Logfile and loglevel can be set in a container config file.    // Those function as defaults.  The defaults can be overriden    // by command line.  However we don't want the command line    // specified values to be saved on c->save_config().  So we    // store the config file specified values here.    char *logfile;  // the logfile as specifed in config    int loglevel;   // loglevel as specifed in config (if any)    int inherit_ns_fd[LXC_NS_MAX];    int start_auto;    int start_delay;    int start_order;    struct lxc_list groups;    int nbd_idx;    /* set to true when rootfs has been setup */    bool rootfs_setup;};

lxc_info 的结构体看完，下面要看下lxc-container 这个重头戏。

下面来看下lxc_container的结构体

/*! * An LXC container. */struct lxc_container {    // private fields   char *name;  //container 的名字   char *configfile;  // configuration file 的路径   char *pidfile;    // 存储pid 的文件名   struct lxc_lock *slock;  //Container semaphore lock. 容器的信号锁    struct lxc_lock *privlock;//容器的私有信号锁    int numthreads;//容器的引用数量，由privlock保护    struct lxc_conf *lxc_conf;    // public fields    char *error_string;//全局变量 可读的最后显示的error    int error_num;//最后error的数字    bool daemonize;//容器是否希望开启守护进程char *config_path;// configuration file 的路径 和上面的区别？ 全局？…….//一堆成员函数   暂不看}

好了， lxc_container暂时看到这个位置，后面需要的话再一个一个的详解

 

lxc_list_init(&defines);             //初始化list

defines定义在文件开始，为全局变量

static structlxc_list defines;

if(lxc_caps_init())                    //caps初始化

        return err;

到这个函数里看一下。

int lxc_caps_init(void){    uid_t uid = getuid();    gid_t gid = getgid();    uid_t euid = geteuid();//有效uid    if (!uid) {//root权限运行的话就省了后面的步骤了        INFO("command is run as 'root'");        return 0;    }       if (uid && !euid) {        INFO("command is run as setuid root (uid : %d)", uid);        if (prctl(PR_SET_KEEPCAPS, 1)) {//prctl 设置进程的选项，为下面set？            ERROR("failed to 'PR_SET_KEEPCAPS': %m");            return -1;         }           if (setresgid(gid, gid, gid)) {            ERROR("failed to change gid to '%d': %m", gid);            return -1;         }           if (setresuid(uid, uid, uid)) {            ERROR("failed to change uid to '%d': %m", uid);            return -1;         }           if (lxc_caps_up()) {            ERROR("failed to restore capabilities: %m");            return -1;         }       }   if (uid == euid)INFO("command is run as user '%d'", uid);return 0;}

接着就是读传过来的参数

if(lxc_arguments_parse(&my_args, argc, argv))

        return err;

这个函数就没细看，只需知道将参数传给my_args

判断有没有指定 初始执行的参数，没有的话指定默认参数

if (!my_args.argc)

        args = default_args;

    else       

        args = my_args.argv;

 

初始化一堆log的，暂时也没细看

if (lxc_log_init(my_args.name, my_args.log_file, my_args.log_priority,             my_args.progname, my_args.quiet, my_args.lxcpath[0]))        return err;lxc_log_options_no_override();const char *lxcpath = my_args.lxcpath[0];//lxcpath 很有意思// lxc_global_config_value("lxc.lxcpath")这个写的还是比较复杂的，总之lxcpath会是默认的路径//指定config的位置，如果没指定，则使用默认的路径的config，通过配置创建新的/*     * rcfile possibilities:     * 1. rcfile from random path specified in cli option     * 2. rcfile not specified, use $lxcpath/$lxcname/config     * 3. rcfile not specified and does not exist.     */    /* rcfile is specified in the cli option */    if (my_args.rcfile) {        rcfile = (char *)my_args.rcfile;        c = lxc_container_new(my_args.name, lxcpath);        if (!c) {            ERROR("Failed to create lxc_container");            return err;        }        c->clear_config(c);        if (!c->load_config(c, rcfile)) {            ERROR("Failed to load rcfile");            lxc_container_put(c);            return err;        }} } else {        int rc;        rc = asprintf(&rcfile, "%s/%s/config", lxcpath, my_args.name);        if (rc == -1) {            SYSERROR("failed to allocate memory");            return err;        }        INFO("using rcfile %s", rcfile);        /* container configuration does not exist */        if (access(rcfile, F_OK)) {            free(rcfile);            rcfile = NULL;        }        c = lxc_container_new(my_args.name, lxcpath);        if (!c) {            ERROR("Failed to create lxc_container");            return err;        }}里面最主要的函数c = lxc_container_new(my_args.name, lxcpath);struct lxc_container *lxc_container_new(const char *name, const char *configpath){    struct lxc_container *c;//结构体lxc_container 前面分析过了    c = malloc(sizeof(*c));//创建    if (!c) {        fprintf(stderr, "failed to malloc lxc_container\n");        return NULL;    }        memset(c, 0, sizeof(*c));   //初始0    if (configpath)        c->config_path = strdup(configpath);//config_path    else         c->config_path = strdup(lxc_global_config_value("lxc.lxcpath"));    if (!c->config_path) {        fprintf(stderr, "Out of memory\n");        goto err;     }        remove_trailing_slashes(c->config_path);    c->name = malloc(strlen(name)+1);    if (!c->name) {        fprintf(stderr, "Error allocating lxc_container name\n");        goto err;     }        strcpy(c->name, name);    c->numthreads = 1;  // lock这部分没细看    if (!(c->slock = lxc_newlock(c->config_path, name))) {        fprintf(stderr, "failed to create lock\n");        goto err;     } if (!(c->privlock = lxc_newlock(NULL, NULL))) {        fprintf(stderr, "failed to alloc privlock\n");        goto err;    } // set config path if (!set_config_filename(c)) {        fprintf(stderr, "Error allocating config file pathname\n");        goto err;    } //load config path    if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL))        goto err; //判断容器是否创建失败 if (ongoing_create(c) == 2) {        ERROR("Error: %s creation was not completed", c->name);        lxcapi_destroy(c);        lxcapi_clear_config(c);    }    c->daemonize = true;    c->pidfile = NULL; …… //后面都是成员函数赋值}现在回到lxc_start 的main函数中//判断容器是否在运行if (c->is_running(c)) {        ERROR("Container is already running.");        err = 0;        goto out; }/*        * We should use set_config_item() over &defines, which would handle  * unset c->lxc_conf for us and let us not use lxc_config_define_load()  *///加载config文件 if (!c->lxc_conf)     c->lxc_conf = lxc_conf_init(); conf = c->lxc_conf;if (lxc_config_define_load(&defines, conf))        goto out;//提示信息if (!rcfile && !strcmp("/sbin/init", args[0])) {        ERROR("Executing '/sbin/init' with no configuration file may crash the host");        goto out;    }    if (ensure_path(&conf->console.path, my_args.console) < 0) {        ERROR("failed to ensure console path '%s'", my_args.console);        goto out;    }    if (ensure_path(&conf->console.log_path, my_args.console_log) < 0) {        ERROR("failed to ensure console log '%s'", my_args.console_log);        goto out;    }// pid 文件    if (my_args.pidfile != NULL) {        if (ensure_path(&c->pidfile, my_args.pidfile) < 0) {            ERROR("failed to ensure pidfile '%s'", my_args.pidfile);            goto out;        }    }    //一些share_ns 的配置，未细看 int i;    for (i = 0; i < LXC_NS_MAX; i++) {        if (my_args.share_ns[i] == NULL)            continue;        int pid = pid_from_lxcname(my_args.share_ns[i], lxcpath);        if (pid < 1)            goto out;        int fd = open_ns(pid, ns_info[i].proc_name);        if (fd < 0)            goto out;        conf->inherit_ns_fd[i] = fd;    }    //初始化为1    if (!my_args.daemonize) {        c->want_daemonize(c, false);    }    if (my_args.close_all_fds)        c->want_close_all_fds(c, true);    err = c->start(c, 0, args) ? 0 : 1;    if (err) {        ERROR("The container failed to start.");        if (my_args.daemonize)            ERROR("To get more details, run the container in foreground mode.");        ERROR("Additional information can be obtained by setting the "              "--logfile and --logpriority options.");        err = c->error_num;        lxc_container_put(c);        return err;    }out:    lxc_container_put(c);    return err;}

直接到c->start 过程start是调用 lxcapi_start 这个函数指针，现在去看下这个函数到底是怎么讲lxc container 启动起来的。

    传过来的参数是container c，useinit 0，argv=args 即指定的初始化程序

static bool lxcapi_start(struct lxc_container *c, int useinit, char * const argv[]){    int ret;    struct lxc_conf *conf;    bool daemonize = false;//守护进程为false    FILE *pid_fp = NULL;//pid_file文件的指针    char *default_args[] = {//又是default_args        "/sbin/init",        NULL,    };    /* container exists */    if (!c)//判断容器是否存在        return false;    /* container has been setup */    if (!c->lxc_conf)//config加载完美        return false;    if ((ret = ongoing_create(c)) < 0) {//容器是否创建完整        ERROR("Error checking for incomplete creation");        return false;    }    if (ret == 2) {        ERROR("Error: %s creation was not completed", c->name);        c->destroy(c);        return false;    } else if (ret == 1) {        ERROR("Error: creation of %s is ongoing", c->name);        return false;    } /* is this app meant to be run through lxcinit, as in lxc-execute? */    if (useinit && !argv)//还是判断        return false;    if (container_mem_lock(c))//lock        return false;    conf = c->lxc_conf;//conf赋值    daemonize = c->daemonize;//true    container_mem_unlock(c);//unlock    if (useinit) {//0        ret = lxc_execute(c->name, argv, 1, conf, c->config_path);        return ret == 0 ? true : false;    }    if (!argv)        argv = default_args;//又重新判断 args 是否为空，空即赋值 /*    * say, I'm not sure - what locks do we want here?  Any?    * Is liblxc's locking enough here to protect the on disk    * container?  We don't want to exclude things like lxc_info    * while container is running... * 这段注释给跪了，还是老老实实看他想干嘛吧    */    if (daemonize) {//true        lxc_monitord_spawn(c->config_path);//start好像跟前面的版本差别        pid_t pid = fork();        if (pid < 0)            return false;        if (pid != 0) {            /* Set to NULL because we don't want father unlink             * the PID file, child will do the free and unlink.             */            c->pidfile = NULL;            return wait_on_daemonized_start(c, pid);//等下进去，里面有waitpid，所以先看后面        }        /* second fork to be reparented by init */        pid = fork();//两次fork        if (pid < 0) {            SYSERROR("Error doing dual-fork");            return false;        }        if (pid != 0)            exit(0);        /* like daemon(), chdir to / and redirect 0,1,2 to /dev/null */        if (chdir("/")) {//root目录            SYSERROR("Error chdir()ing to /.");            return false;        }        lxc_check_inherited(conf, -1);        close(0);//pipe file?close(1);        close(2);        open("/dev/zero", O_RDONLY);        open("/dev/null", O_RDWR);        open("/dev/null", O_RDWR);        setsid();    } else {        if (!am_single_threaded()) {            ERROR("Cannot start non-daemonized container when threaded");            return false;        }    }/* We need to write PID file after daeminize, so we always     * write the right PID.     */    if (c->pidfile) {//写入pid 到pidfile        pid_fp = fopen(c->pidfile, "w");        if (pid_fp == NULL) {            SYSERROR("Failed to create pidfile '%s' for '%s'",                 c->pidfile, c->name);            return false;        }        if (fprintf(pid_fp, "%d\n", getpid()) < 0) {            SYSERROR("Failed to write '%s'", c->pidfile);            fclose(pid_fp);            pid_fp = NULL;            return false;        }        fclose(pid_fp);        pid_fp = NULL;    }reboot:…..     }

现在到 wait_on_daemonized_start(c, pid) 里面看看函数调用的情况

这个就是主线程的pid 在等待其他子线程工作完，然后执行，只能硬着头皮继续看了。

static bool wait_on_daemonized_start(struct lxc_container *c, int pid){    /* we'll probably want to make this timeout configurable? */    int timeout = 5, ret, status;    /*     * our child is going to fork again, then exit.  reap the     * child     */    ret = waitpid(pid, &status, 0);    if (ret == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)        DEBUG("failed waiting for first dual-fork child");    return lxcapi_wait(c, "RUNNING", timeout);}

函数很简单 直接调用了lxcapi_wait。

static bool lxcapi_wait(struct lxc_container *c, const char *state, int timeout){    int ret;    if (!c)        return false;    ret = lxc_wait(c->name, state, timeout, c->config_path);    return ret == 0;}

这个依旧很简单又跳走了。。。lxc_wait了

这个函数现在先不细说了，只是检查容器创建是否超时的问题。

 

Reboot这货还是很奇葩，我们这一代都在灌输少用或者不用goto，导致我见到这个一直很疑惑goto在哪，结果找了半天没找到，好吧 他是单独的一个程序块，瞅瞅去，重头戏都在这里。

ok 下篇再看reboot吧，重头戏都在里面