C++ SOCKET通信模型（七）异步epoll

不得不说为这异步epoll模型还真是伤了神。。。主要问题就是在libaio的内核通知上，因为这东西实在太低级了，用起来还比较麻烦。我为了不再多开线程，实现和IOCP基本相同的原理，在PROC内部又用了次epoll，使其内核通知和用户通知都能在一个线程中得到相同处理。并且用map映射eventfd与收发类型的关系，使其能够在收到event的时候，通过eventfd能直接判断出接下来的操作。还有就是由于libaio需要将fd设置为O_DIRECT，所以无论收发都得用自己的缓冲区，不然会导致效率大大降低。这份代码有兴趣自己研究，具体的不多说了，写出来还没测试，不知道应用于实际项目中效果如何，大牛可以自己测下

yum install libaio-devel

-lpthread -std=c++11

#include <iostream>#include<list>#include<map>#include<deque>#include<mutex>#include <atomic>#include<condition_variable>#include<sys/socket.h>#include<sys/epoll.h>#include<netinet/in.h>#include <pthread.h>#include<unistd.h>#include<fcntl.h>#include<signal.h>#include<libaio.h>#include<sys/eventfd.h>#define RECV_EVENTS 1024#define SEND_EVENTS 1024#define PROC_EVENTS 1024#define BUF_SEND_LEN 1024#define BUF_RECV_LEN 1024#define RECV_LEN 128#define SEND_LEN 128using namespace std;typedef unsigned long long ull;struct Client{iocb cb;int id;int fd;char* recvbuf;char* sendbuf;};void* CreateServ(void* args);void* Proc(void* args);using namespace std;const int _thread_count = 8;int _epfd;int sockSrv;int id = 0;epoll_event _events[RECV_EVENTS] = { 0 };epoll_event ev[_thread_count];mutex m;deque<epoll_event*> _deque;map<int, Client*> _sockMap;list<int> _removeList;mutex lock4cv2;condition_variable cv2;int _thread_unfinish;int _eventfd4send[_thread_count];int _eventfd4recv[_thread_count];int _proc_epfd[_thread_count];map<int, int> _eventfd_type_map;enum{RECV, SEND};int main(){pthread_t tid;for (int i = 0; i < _thread_count; i++){_proc_epfd[i] = epoll_create(PROC_EVENTS);_eventfd4send[i] = eventfd(0, 0);ev[0].data.fd = _eventfd4send[i];ev[0].events = EPOLLIN | EPOLLET;epoll_ctl(_proc_epfd[i], EPOLL_CTL_ADD, _eventfd4send[i], &ev[0]);_eventfd_type_map.insert(pair<int, int>(_eventfd4send[i], SEND));_eventfd4recv[i] = eventfd(0, 0);ev[0].data.fd = _eventfd4recv[i];ev[0].events = EPOLLIN | EPOLLET;epoll_ctl(_proc_epfd[i], EPOLL_CTL_ADD, _eventfd4recv[i], &ev[0]);_eventfd_type_map.insert(pair<int, int>(_eventfd4recv[i], RECV));}pthread_create(&tid, 0, CreateServ, 0);for (int i = 0; i < _thread_count; i++){int* temp = new int(i);pthread_create(&tid, 0, Proc, temp);}cin.get();cin.get();return 0;}bool _isFinish(){return _thread_unfinish <= 0;}void release(int fd){cout << "release:" << fd << endl;ev[0].data.fd = fd;epoll_ctl(_epfd, EPOLL_CTL_DEL, fd, &ev[0]);for (auto iter = _sockMap.begin(); iter != _sockMap.end(); ++iter){if (iter->first == fd){free(iter->second->recvbuf);free(iter->second->sendbuf);delete iter->second;_sockMap.erase(iter);break;}}close(fd);}static void recv_callback(io_context_t ctx, struct iocb *iocb, long res, long res2){Client* c = (Client*)iocb;//cout << c->recvbuf << endl;memset(c->recvbuf, 0, sizeof(char)* BUF_RECV_LEN);sprintf(c->sendbuf, "hello client");io_prep_pwrite(&c->cb, c->fd, c->sendbuf, SEND_LEN, 0);}void* Proc(void* args){int* I = (int*)args;int events_num = 0;epoll_event threadEvents[PROC_EVENTS];io_context_t ioctx;io_event events[RECV_EVENTS];iocb* io_submit_arr[RECV_EVENTS];io_queue_init(RECV_EVENTS, &ioctx);io_context_t ioctx4write;iocb* io_submit_arr2[SEND_EVENTS];io_queue_init(SEND_EVENTS, &ioctx4write);while (true){events_num = epoll_wait(_proc_epfd[*I], threadEvents, PROC_EVENTS, -1);for (int i = 0; i < events_num; ++i) {if (threadEvents[i].events == EPOLLIN){int t = _eventfd_type_map[threadEvents[i].data.fd];if (t == RECV){int i = 0;while (true){m.lock();if (_deque.size() == 0){m.unlock();break;}epoll_event* e = _deque.front();_deque.pop_front();m.unlock();if (e->data.fd == -1){continue;}if (e->data.fd == sockSrv){}else if (e->events == EPOLLIN){Client* c = _sockMap[e->data.fd];io_prep_pread(&c->cb, e->data.fd, c->recvbuf, RECV_LEN, 0);io_set_eventfd(&c->cb, _eventfd4send[*I]);io_set_callback(&c->cb, recv_callback);io_submit_arr[i] = &c->cb;i++;e->events = 0;}else{Client* c = _sockMap[e->data.fd];_removeList.push_back(c->fd);io_cancel(ioctx4write, &c->cb, nullptr);io_cancel(ioctx, &c->cb, nullptr);}}io_submit(ioctx, i, io_submit_arr);//此处一定要加锁。当wait还未准备好，但_isFinish刚刚检测完时_thread_unfinish -= 1;cv2.notify_all();这两句恰巧在之后执行，那就GG了，两边进入互相等待的状态lock4cv2.lock();_thread_unfinish -= 1;cv2.notify_all();lock4cv2.unlock();}else{ull num = 0;read(threadEvents[i].data.fd, &num, sizeof(ull));while (num > 0){timespec tms;tms.tv_sec = 0;tms.tv_nsec = 0;io_getevents(ioctx4write, 1, SEND_EVENTS, events, &tms);int r = io_getevents(ioctx, 1, RECV_EVENTS, events, &tms);if (r > 0){int i = 0;for (; i < r; ++i){((io_callback_t)(events[i].data))(ioctx, events[i].obj, events[i].res, events[i].res2);Client* c = (Client*)events[i].obj;io_submit_arr2[i] = &c->cb;}io_submit(ioctx4write, i, io_submit_arr2);}num -= r;}}}threadEvents[i].events = 0;}}}void* CreateServ(void* args) {ull num = 1;int events_num = 0;sockSrv = socket(AF_INET, SOCK_STREAM, 0);//int nSendBufLen = 16 * 1024 * 1024;//setsockopt(sockSrv, SOL_SOCKET, SO_SNDBUF, (const char*)&nSendBufLen, sizeof(int));fcntl(sockSrv, F_SETFL, O_NONBLOCK);//struct in_addr s;//inet_pton(AF_INET, "127.0.0.1",(void*)&s);sockaddr_in addrSrv;addrSrv.sin_addr.s_addr = htonl(INADDR_ANY);addrSrv.sin_family = AF_INET;addrSrv.sin_port = htons(6001);::bind(sockSrv, (sockaddr*)&addrSrv, sizeof(sockaddr));int err = listen(sockSrv, 1024);if (err == -1) {cout << "listen failed" << endl;return 0;}_epfd = epoll_create(RECV_EVENTS);ev[0].data.fd = sockSrv;ev[0].events = EPOLLIN | EPOLLET;epoll_ctl(_epfd, EPOLL_CTL_ADD, sockSrv, &ev[0]);//accept loopwhile (true) {events_num = epoll_wait(_epfd, _events, RECV_EVENTS, -1);if (events_num<0){continue;}while (true) {int s = accept(sockSrv, 0, 0);if (s <= 0){break;}fcntl(s, F_SETFL, O_NONBLOCK | O_DIRECT);Client* c = new Client;posix_memalign((void**)&c->recvbuf, getpagesize(), BUF_RECV_LEN);posix_memalign((void**)&c->sendbuf, getpagesize(), BUF_SEND_LEN);c->id = id;c->fd = s;id += 1;_sockMap.insert(pair<int, Client*>(s, c));ev[0].data.fd = s;ev[0].events = EPOLLIN | EPOLLET;epoll_ctl(_epfd, EPOLL_CTL_ADD, s, &ev[0]);}_thread_unfinish = _thread_count;for (int i = 0; i<events_num; i++){_deque.push_back(&_events[i]);}for (int i = 0; i<_thread_count; i++){write(_eventfd4recv[i], &num, sizeof(ull));}unique_lock<mutex> l(lock4cv2);cv2.wait(l, _isFinish);for (auto iter = _removeList.begin(); iter != _removeList.end(); ++iter){release(*iter);id -= 1;}_removeList.clear();}}