host_notifier, 虚拟机通过VHOST发包流程(基于kernel3.10.0 && qemu 2.0.0)

1. 系统中的eventfd_add(), 以及memory_listener的注册

static MemoryListener kvm_memory_listener = {

    .eventfd_add = kvm_mem_ioeventfd_add,

};

static MemoryListener kvm_io_listener = {

    .eventfd_add = kvm_io_ioeventfd_add,

};

int kvm_init(QEMUMachine *machine){

    memory_listener_register(&kvm_memory_listener, &address_space_memory);

    memory_listener_register(&kvm_io_listener, &address_space_io);

}

net_init_tap

-> net_init_tap_one

-> vhost_net_init

-> vhost_dev_init

int vhost_dev_init(){

    hdev->memory_listener = (MemoryListener) {

        .eventfd_add = vhost_eventfd_add,

    };

    ...

    memory_listener_register(&hdev->memory_listener, &address_space_memory);

}

2. virtio pci设备的读写操作注册

static const MemoryRegionOps virtio_pci_config_ops = {

    .write = virtio_pci_config_write,

};

3. eventfd_add的触发，用于向内核中注册VIRTIO PCI设备

Qemu截获寄存器的访问，调用注册的kvm_memory_listener中的eventfd_add回调函数kvm_eventfd_add()；

virtio_pci_config_ops->write //调用virtio_pci_config_write

-> virtio_pci_config_write

-> virtio_ioport_write

-> virtio_pci_start_ioeventfd

-> virtio_pci_set_host_notifier_internal

-> memory_region_add_eventfd

-> memory_region_transaction_commit

-> address_space_update_topology

-> address_space_update_ioeventfds

-> address_space_add_del_ioeventfds

address_space_add_del_ioeventfds(){

MEMORY_LISTENER_CALL(eventfd_add, ......); //调用eventadd, 起作用的是kvm_io_ioeventfd_add

}

3. kvm_io_ioeventfd_add将事件下发到内核，初始化注册VIRTIO_PCI的IO设备

eventadd

-> kvm_io_ioeventfd_add

-> kvm_set_ioeventfd_pio

-> r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);

5. KVM模块响应并初始化增加IO_DEVICE的eventfd

kvm_vm_ioctl

-> kvm_ioeventfd 

-> kvm_assign_ioeventfd //创建IO设备的，并分配eventfd

-> kvm_iodevice_init(&p->dev, &ioeventfd_ops);  //初始化write方法为ioeventfd_write

-> kvm_io_bus_register_dev() 

-> list_add_tail(&p->list, &kvm->ioeventfds); 

static const struct kvm_io_device_ops ioeventfd_ops = {

.write      = ioeventfd_write,

};

6.VHOST内核线程的建立

vhost_net_ioctl() //VHOST_SET_OWNER

-> vhost_net_set_owner

-> vhost_dev_set_owner

-> worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); //处理函数为vhost_worker

7.VHOST设备进行收发poll的注册，当被open的时候，注册读写的处理函数为handle_tx_net和handle_tx_net，并且使用默认的poll方法

对vhost-net文件进行open时

static const struct file_operations vhost_net_fops = {

.open           = vhost_net_open,

//使用默认的poll方法，？？？怎么和eventfd_fops联系起来的？

};

默认的poll方法

static const struct file_operations eventfd_fops = {

.poll= eventfd_poll,

};

static struct miscdevice vhost_net_misc = {

.name = "vhost-net",

.fops = &vhost_net_fops,

};

/*

 * 在vhost_net_open()中对vhost进行了初始化

 */

static int vhost_net_open(struct inode *inode, struct file *f)

{

n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;

n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;

vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);

vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);

}

vhost_poll_init(){

init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);

init_poll_funcptr(&poll->table, vhost_poll_func); //注册poll的proc方法为vhost_poll_func

}

void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, unsigned long mask, struct vhost_dev *dev)

{

init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);

init_poll_funcptr(&poll->table, vhost_poll_func);

vhost_work_init(&poll->work, fn);

}

vhost_net_ioctl //default

-> vhost_vring_ioctl

-> vhost_poll_start

-> file->f_op->poll(file, &poll->table); //？？怎么和eventfd联系起来的，这里用默认的eventfd_poll，会阻塞在这里，一直等到有事件才返回vhost_poll_start

-> eventfd_poll

-> poll_wait

-> p->_qproc(filp, wait_address, p); //这里是vhost_poll_func

-> vhost_poll_func

-> add_wait_queue

8. KVM模块响应IO设备写操作

当GUEST有IO设备写操作，会触发write,也就是触发ioeventfd_write

ioeventfd_write

-> eventfd_signal

-> wake_up_locked_poll(&ctx->wqh, POLLIN); //这里通知到了vhost work thread

9. vhost模块内核线程报文收发流程

vhost的线程在vhost_poll_start里阻塞在file->f_op->poll(file, &poll->table)，直到wake_up_locked_poll(&ctx->wqh, POLLIN);后才返回

vhost_poll_start //从mask = file->f_op->poll(file, &poll->table);返回

-> vhost_poll_wakeup

-> vhost_poll_wakeup

-> vhost_poll_queue

-> vhost_work_queue

-> wake_up_process(dev->worker); //让worker运行，每个worker是一个vhost的内核线程，这里的处理函数为vhost_worker 

-> vhost_worker

-> work->fn(work); //fn就是open时注册的handle_tx_net和handle_rx_net, 这里调用fn进行收发

handle_rx_net //收包

-> handle_rx

-> sock->ops->recvmsg(NULL, sock, &msg, sock_len, MSG_DONTWAIT | MSG_TRUNC); //标准socket接收方法

handle_tx_net //发包

-> handle_tx

-> sock->ops->sendmsg(NULL, sock, &msg, len);