epoll + 线程池 回应服务器(Linux) －－－可以支持上万个连接

//ptypes#include "ptime.h"#include "pinet.h"#include "ptypes.h"#include "pasync.h"#include "logfile.h"//network#include <sys/socket.h>#include <sys/epoll.h>#include <netinet/in.h>#include <arpa/inet.h>#include <fcntl.h>#include <unistd.h>#include <stdio.h>#include <errno.h>#include <iostream>#include <set>using namespace std;USING_PTYPES#define MAX_EVENTS 10000#define maxtoken 1024const int maxthreads = 64;const int MSG_MYJOB = 100;#define BUFF_MAX 1 //现在不用到事件里的缓存区，先设定最小值struct myevent_s{int fd;void (*call_back)(int fd, int events, void *arg);int events;void *arg;int status; // 1: in epoll wait list, 0 not inchar buff[BUFF_MAX]; // recv data bufferint len;long last_active; // last active time};//以Socket为关键字的集合.std::set<int> m_set;//增加锁定。mutex m_mxLock;//把socket插入到集合中,如果存在返回falsebool  InsertSocket(int nSocket){scopelock lock(m_mxLock);if (m_set.end() == m_set.find(nSocket)){//没有找到插入并返回false；m_set.insert(nSocket);return true;}else{//已经存在不插入返回falsereturn false;}}//重集合中删除socket;bool  EraseSocket(int nSocket){scopelock lock(m_mxLock);//如果集合中存在socket,把Socket从集合里删除if (m_set.end() != m_set.find(nSocket)){//找到再删除；m_set.erase(nSocket);return true;}else{//不存在不删除return false;}}// set eventvoid EventSet(myevent_s *ev, int fd, void (*call_back)(int, int, void*), void *arg){ev->fd = fd;ev->call_back = call_back;ev->events = 0;ev->arg = arg;ev->status = 0;ev->last_active = time(NULL);}// add/mod an event to epollvoid EventAdd(int epollFd, int events, myevent_s *ev){struct epoll_event epv = {0, {0}};int op;epv.data.ptr = ev;epv.events = ev->events = events;if(ev->status == 1){op = EPOLL_CTL_MOD;}else{op = EPOLL_CTL_ADD;ev->status = 1;}if(epoll_ctl(epollFd, op, ev->fd, &epv) < 0){printf("Event Add failed[fd=%d], status:%d\n", ev->fd, ev->status);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: Event Add failed[fd=%d], status:%d error=%d %s\n",  __func__, ev->fd, ev->status,errno, strerror(errno));}else{printf("Event Add OK[fd=%d]\n", ev->fd);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: Event Add OK[fd=%d]\n", __func__,  ev->fd);}}// delete an event from epollvoid EventDel(int epollFd, myevent_s *ev){struct epoll_event epv = {0, {0}};if(ev->status != 1) return;epv.data.ptr = ev;ev->status = 0;epoll_ctl(epollFd, EPOLL_CTL_DEL, ev->fd, &epv);}int g_epollFd;myevent_s g_Events[MAX_EVENTS+1]; // g_Events[MAX_EVENTS] is used by listen fdvoid RecvData(int fd, int events, void *arg);void SendData(int fd, int events, void *arg);// accept new connections from clientsvoid AcceptConn(int fd, int events, void *arg){struct sockaddr_in sin;socklen_t len = sizeof(struct sockaddr_in);int nfd, i;// acceptif((nfd = accept(fd, (struct sockaddr*)&sin, &len)) == -1){if(errno != EAGAIN && errno != EINTR){printf("%s: bad accept", __func__);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: bad accept error = %d %s", __func__, errno, strerror(errno) );}return;}do{for(i = 0; i < MAX_EVENTS; i++){if(g_Events[i].status == 0){break;}}if(i == MAX_EVENTS){printf("%s:max connection limit[%d].", __func__, MAX_EVENTS);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: max connection limit[%d]; error = %d %s", __func__, MAX_EVENTS, errno, strerror(errno) );break;}// set nonblockingif(fcntl(nfd, F_SETFL, O_NONBLOCK) < 0) break;// add a read event for receive dataEventSet(&g_Events[i], nfd, RecvData, &g_Events[i]);EventAdd(g_epollFd, EPOLLIN|EPOLLET, &g_Events[i]);printf("new conn[%s:%d][time:%d]\n", inet_ntoa(sin.sin_addr), ntohs(sin.sin_port), g_Events[i].last_active);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: new conn[%s:%d][time:%d]", __func__, inet_ntoa(sin.sin_addr), ntohs(sin.sin_port), g_Events[i].last_active);}while(0);}// receive datavoid RecvData(int fd, int events, void *arg){struct myevent_s *ev = (struct myevent_s*)arg;int len;// receive datalen = recv(fd, ev->buff, sizeof(ev->buff)-1, 0); CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: recv len:%d", __func__, len);EventDel(g_epollFd, ev);if(len > 0){ev->len = len;ev->buff[len] = '\0';printf("C[%d]:%s\n", fd, ev->buff);// change to send eventEventSet(ev, fd, SendData, ev);EventAdd(g_epollFd, EPOLLOUT|EPOLLET, ev);}else if(len == 0){close(ev->fd);printf("[fd=%d] closed gracefully.\n", fd);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: closed gracefully", __func__);}else{close(ev->fd);printf("recv[fd=%d] error[%d]:%s\n", fd, errno, strerror(errno));CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: recv[fd=%d] error[%d]:%s", __func__, fd, errno, strerror(errno));}}// send datavoid SendData(int fd, int events, void *arg){struct myevent_s *ev = (struct myevent_s*)arg;int len;// send datalen = send(fd, ev->buff, ev->len, 0);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: send len:%d", __func__, len);ev->len = 0;EventDel(g_epollFd, ev);if(len > 0){// change to receive eventEventSet(ev, fd, RecvData, ev);EventAdd(g_epollFd, EPOLLIN|EPOLLET, ev);}else{close(ev->fd);printf("send[fd=%d] error[%d]\n", fd, errno);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, " %s: send[fd=%d] error[%d]:%s", __func__, fd, errno, strerror(errno));}}void InitListenSocket(int epollFd, short port){int listenFd = socket(AF_INET, SOCK_STREAM, 0);fcntl(listenFd, F_SETFL, O_NONBLOCK); // set non-blockingprintf("server listen fd=%d/n", listenFd);EventSet(&g_Events[MAX_EVENTS], listenFd, AcceptConn, &g_Events[MAX_EVENTS]);// add listen socket//EventAdd(epollFd, EPOLLIN|EPOLLET, &g_Events[MAX_EVENTS]);EventAdd(epollFd, EPOLLIN, &g_Events[MAX_EVENTS]);// bind & listensockaddr_in sin;bzero(&sin, sizeof(sin));sin.sin_family = AF_INET;sin.sin_addr.s_addr = INADDR_ANY;sin.sin_port = htons(port);bind(listenFd, (const sockaddr*)&sin, sizeof(sin));listen(listenFd, SOMAXCONN);}//线程池函数class myjobthread: public thread{protected:int id;jobqueue* jq;virtual void execute();public:myjobthread(int iid, jobqueue* ijq): thread(false), id(iid), jq(ijq)  {}~myjobthread()  { waitfor(); }};class myjob: public message{public:myevent_s* m_pSocketInfo;myjob(myevent_s* pSocketInfo): message(MSG_MYJOB), m_pSocketInfo(pSocketInfo)  {}~myjob()  {//delete m_pSocketInfo;}};void myjobthread::execute(){bool quit = false;while (!quit){// get the next message from the queuemessage* msg = jq->getmessage();try{switch (msg->id){case MSG_MYJOB:{pout.putf("%t,线程：%d\n", now(), id);myevent_s *ev = ((myjob*)msg)->m_pSocketInfo;if(ev->events&EPOLLIN) // read event{ev->call_back(ev->fd, ev->events, ev->arg);}if(ev->events&EPOLLOUT) // write event{ev->call_back(ev->fd, ev->events, ev->arg);}//pout.putf("%t,%d返回:%d字节\n",now(), id, nSend);}break;case MSG_QUIT:// MSG_QUIT is not used in our examplequit = true;break;}}catch(pt::exception*){// the message object must be freed!delete msg;throw;}delete msg;}}int main(int argc, char **argv){short port = 8108; // default portif(argc == 2){port = atoi(argv[1]);}CLogFile::Instance()->LOG_Start("./log/");//启动线程池jobqueue jq;tobjlist<myjobthread> threads(true);// create the thread poolfor(int n = 0; n < maxthreads; n++){myjobthread* j = new myjobthread(n + 1, &jq);j->start();threads.add(j);}// create epollg_epollFd = epoll_create(MAX_EVENTS);if(g_epollFd <= 0) {CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "create epoll failed.");printf("create epoll failed.%d\n", g_epollFd);}// create & bind listen socket, and add to epoll, set non-blockingInitListenSocket(g_epollFd, port);// event loopstruct epoll_event events[MAX_EVENTS];printf("server running:port[%d]\n", port);int checkPos = 0;while(1){// a simple timeout check here, every time 100, better to use a mini-heap, and add timer eventlong now = time(NULL);for(int k = 0; k < MAX_EVENTS; k++, checkPos++) // doesn't check listen fd{if(checkPos == MAX_EVENTS) checkPos = 0; // recycleif(g_Events[checkPos].status != 1) continue;long duration = now - g_Events[checkPos].last_active;if(duration >= 600) // 600s timeout{closesocket(g_Events[checkPos].fd);printf("[fd=%d] timeout[%d--%d].\n", g_Events[checkPos].fd, g_Events[checkPos].last_active, now);CLogFile::Instance()->LOG_WriteLine(LOG_INFO_WARN, "超时Socket：[fd=%d] timeout[%d--%d].", g_Events[checkPos].fd, g_Events[checkPos].last_active, now);EventDel(g_epollFd, &g_Events[checkPos]);}}// wait for events to happenintfds = epoll_wait(g_epollFd, events, MAX_EVENTS, -1);if(fds < 0){printf("epoll_wait error, exit/n");CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "epoll_wait error.");break;}CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "EPOLL_WAIT FDS counts:%d", fds);for(int i = 0; i < fds; i++){myevent_s *ev = (struct myevent_s*)events[i].data.ptr;jq.post(new myjob(ev));}}// free resourcereturn 0;}