#include <stdio.h>#include <stdlib.h>#include <errno.h>#include <string.h>#include <sys/types.h>#include <netinet/in.h>#include <sys/socket.h>#include <sys/wait.h>#include <unistd.h>#include <arpa/inet.h>//#include <openssl/ssl.h>//#include <openssl/err.h>#include <fcntl.h>#include <sys/epoll.h>#include <sys/time.h>#include <sys/resource.h>#include <pthread.h>#include <assert.h>//#define DEBUG_TILERA#ifdef DEBUG_TILERA#include <tmc/alloc.h>#include <arch/cycle.h>#include <arch/spr.h>#include <tmc/cpus.h>#include <tmc/sync.h>#include <tmc/task.h>#endif /* These are non-NULL pointers that will result in page faults * under normal circumstances, used to verify that nobody uses * non-initialized list entries. */#define LIST_POISON1 ((void *) 0x00100100)#define LIST_POISON2 ((void *) 0x00200200)#define MAXBUF 1024#define MAXEPOLLSIZE 100000#define MAX_THREAD_NUMBER 200int THREAD_NUMBER = 50;int kdpfd;struct epoll_event events[MAXEPOLLSIZE];int msgcount = 0;int timecount = 0;int count_packet= 0;int connect_count = 0;pthread_mutex_t curfds_lock;int curfds;char buffer[MAX_THREAD_NUMBER][MAXBUF + 1];pthread_t thread_count;cpu_set_t cpus;#define offset_of(type, memb) \((unsigned long)(&((type *)0)->memb))#define container_of(obj, type, memb) \((type *)(((char *)obj) - offset_of(type, memb)))typedef struct list_head { struct list_head *next, *prev;}list_head;#define LIST_HEAD_INIT(name) { &(name), &(name) }#define LIST_HEAD(name) \ struct list_head name = LIST_HEAD_INIT(name)static inline void INIT_LIST_HEAD(struct list_head *list){ list->next = list; list->prev = list;}/* * Insert a new entry between two known consecutive entries. * * This is only for internal list manipulation where we know * the prev/next entries already! */static inline void __list_add(struct list_head *new, struct list_head *prev, struct list_head *next){ next->prev = new; new->next = next; new->prev = prev; prev->next = new;}/** * list_add - add a new entry * @new: new entry to be added * @head: list head to add it after * * Insert a new entry after the specified head. * This is good for implementing stacks. */static inline void list_add(struct list_head *new, struct list_head *head){ __list_add(new, head, head->next);}/** * list_add_tail - add a new entry * @new: new entry to be added * @head: list head to add it before * * Insert a new entry before the specified head. * This is useful for implementing queues. */static inline void list_add_tail(struct list_head *new, struct list_head *head){ __list_add(new, head->prev, head);}/* * Delete a list entry by making the prev/next entries * point to each other. * * This is only for internal list manipulation where we know * the prev/next entries already! */static inline void __list_del(struct list_head * prev, struct list_head * next){ next->prev = prev; prev->next = next;}static inline void list_del(struct list_head *entry){ __list_del(entry->prev, entry->next); entry->next = LIST_POISON1; entry->prev = LIST_POISON2;}/** * list_entry - get the struct for this entry * @ptr: the &struct list_head pointer. * @type: the type of the struct this is embedded in. * @member: the name of the list_struct within the struct. */#define list_entry(ptr, type, member) \ container_of(ptr, type, member)/** * list_first_entry - get the first element from a list * @ptr: the list head to take the element from. * @type: the type of the struct this is embedded in. * @member: the name of the list_struct within the struct. * * Note, that list is expected to be not empty. */#define list_first_entry(ptr, type, member) \ list_entry((ptr)->next, type, member)/** * list_for_each - iterate over a list * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. */#define list_for_each(pos, head) \ for (pos = (head)->next; pos != (head); \ pos = pos->next)/** * list_for_each_safe - iterate over a list safe against removal of list entry * @pos: the &struct list_head to use as a loop cursor. * @n: another &struct list_head to use as temporary storage * @head: the head for your list. */#define list_for_each_safe(pos, n, head) \ for (pos = (head)->next, n = pos->next; pos != (head); \ pos = n, n = pos->next)/**线程池里所有运行和等待的任务都是一个CThread_worker*由于所有任务都在链表里，所以是一个链表结构*/typedef struct worker{ /*回调函数，任务运行时会调用此函数，注意也可声明成其它形式*/ void *(*process) ( int arg, int i); int arg;/*回调函数的参数*/ //struct worker *next;} CThread_worker;typedef struct worker_list{ CThread_worker work; list_head m_list;}worker_list;/*线程池结构*/typedef struct{ pthread_mutex_t queue_lock; pthread_cond_t queue_ready; /*链表结构，线程池中所有等待任务*/ list_head queue_head; /*是否销毁线程池*/ int shutdown; pthread_t *threadid; /*线程池中允许的活动线程数目*/ int max_thread_num; /*当前等待队列的任务数目*/ int cur_queue_size;} CThread_pool;int pool_add_worker (void *(*process) ( int arg, int buff_index), int arg);void *thread_routine (void *arg);static CThread_pool *pool = NULL;voidpool_init (int max_thread_num){ int ret; pool = (CThread_pool *) malloc (sizeof (CThread_pool)); pthread_mutex_init (&(pool->queue_lock), NULL); pthread_cond_init (&(pool->queue_ready), NULL); INIT_LIST_HEAD(&(pool->queue_head)); pool->max_thread_num = max_thread_num; pool->cur_queue_size = 0; pool->shutdown = 0; pool->threadid = (pthread_t *) malloc (max_thread_num * sizeof (pthread_t)); int i = 0; for (i = 0; i < max_thread_num; i++) { ret = pthread_create (&(pool->threadid[i]), NULL, &thread_routine, (void*)i); if(0 != ret) { printf("create thread fail id %d, error %d \n", i, ret); } }}/*向线程池中加入任务*/intpool_add_worker (void *(*process) (int arg, int buff_index), int fd){ /*构造一个新任务*/ worker_list *newworker = (worker_list *) malloc (sizeof (worker_list)); newworker->work.process = process; newworker->work.arg = fd; //newworker->next = NULL;/*别忘置空*/ pthread_mutex_lock (&(pool->queue_lock)); /*将任务加入到等待队列中*/ list_add_tail(&(newworker->m_list), &(pool->queue_head)); /* CThread_worker *member = pool->queue_head; if (member != NULL) { while (member->next != NULL) member = member->next; member->next = newworker; } else { pool->queue_head = newworker; } assert (pool->queue_head != NULL);*/ pool->cur_queue_size++; pthread_mutex_unlock (&(pool->queue_lock)); /*好了，等待队列中有任务了，唤醒一个等待线程； 注意如果所有线程都在忙碌，这句没有任何作用*/ pthread_cond_signal (&(pool->queue_ready)); return 0;}/*销毁线程池，等待队列中的任务不会再被执行，但是正在运行的线程会一直把任务运行完后再退出*/intpool_destroy (){ list_head *p, *tmp; worker_list *tmpwork; if (pool->shutdown) return -1;/*防止两次调用*/ pool->shutdown = 1; /*唤醒所有等待线程，线程池要销毁了*/ pthread_cond_broadcast (&(pool->queue_ready)); /*阻塞等待线程退出，否则就成僵尸了*/ int i; for (i = 0; i < pool->max_thread_num; i++) pthread_join (pool->threadid[i], NULL); free (pool->threadid); /*销毁等待队列*/ list_for_each_safe(p,tmp, &(pool->queue_head)) { tmpwork = list_entry(p, worker_list, m_list); list_del(p); free(tmpwork); } /* CThread_worker *head = NULL; while (pool->queue_head != NULL) { head = pool->queue_head; pool->queue_head = pool->queue_head->next; free (head); } */ /*条件变量和互斥量也别忘了销毁*/ pthread_mutex_destroy(&(pool->queue_lock)); pthread_cond_destroy(&(pool->queue_ready)); free (pool); /*销毁后指针置空是个好习惯*/ pool=NULL; return 0;}void *thread_routine (void *arg){ //printf ("starting thread 0x%x\n", pthread_self ()); //printf("the input para %d \n", ((int)arg));#ifdef DEBUG_TILERA if (tmc_cpus_set_my_cpu(tmc_cpus_find_nth_cpu(&cpus, rank)) < 0) { printf("tmc_cpus_set_my_cpu() failed.\n"); tmc_task_die("tmc_cpus_set_my_cpu() failed."); }#endif while (1) { pthread_mutex_lock (&(pool->queue_lock)); /*如果等待队列为0并且不销毁线程池，则处于阻塞状态; 注意 pthread_cond_wait是一个原子操作，等待前会解锁，唤醒后会加锁*/ while (pool->cur_queue_size == 0 && !pool->shutdown) { // printf ("thread 0x%x is waiting\n", pthread_self ()); pthread_cond_wait (&(pool->queue_ready), &(pool->queue_lock)); } /*线程池要销毁了*/ if (pool->shutdown) { /*遇到break,continue,return等跳转语句，千万不要忘记先解锁*/ pthread_mutex_unlock (&(pool->queue_lock)); printf ("thread 0x%x will exit\n", pthread_self ()); pthread_exit (NULL); } //printf ("thread 0x%x is starting to work\n", pthread_self ()); /*assert是调试的好帮手*/ assert (pool->cur_queue_size != 0); //assert (pool->queue_head != NULL); /*等待队列长度减去1，并取出链表中的头元素*/ pool->cur_queue_size--; worker_list *workerlist; workerlist = list_first_entry(&(pool->queue_head), worker_list, m_list); /*delete it from the list*/ list_del(&(workerlist->m_list)); //pool->queue_head = worker->next; pthread_mutex_unlock (&(pool->queue_lock)); CThread_worker* worker = &(workerlist->work); /*调用回调函数，执行任务*/ (*(worker->process)) (worker->arg, ((int) arg)); free (workerlist); worker = NULL; } /*这一句应该是不可达的*/ pthread_exit (NULL);}/*setnonblocking - 设置句柄为非阻塞方式*/int setnonblocking(int sockfd){ if (fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFD, 0)|O_NONBLOCK) == -1) { return -1; } return 0;}/*handle_message - 处理每个 socket 上的消息收发*/void * handle_message(int new_fd, int buff_index){ char* buf = (char*)&buffer[buff_index]; int len; struct epoll_event ev; /* 开始处理每个新连接上的数据收发 */ //bzero(buf, MAXBUF + 1); /* 接收客户端的消息 */ while((-1 != (len = recv(new_fd, buf, MAXBUF, 0))) ||((-1 == len) &&(EAGAIN != errno))) { //perror("recv error "); //printf("recv error %d fd %d\n", errno, new_fd); //return NULL; if (len > 0) { msgcount++; /*printf ("%d receive message success total %d bytes data msgcount %d\n", new_fd, len, msgcount);*/ } else if(len == 0) { //printf("the socket %d is closed \n", new_fd); epoll_ctl(kdpfd, EPOLL_CTL_DEL, new_fd,&ev); close(new_fd); connect_count--; } else { if (len < 0) printf (" socket %d receive message fail error code: %d, error message: '%s'\n", new_fd, errno, strerror(errno)); epoll_ctl(kdpfd, EPOLL_CTL_DEL, new_fd,&ev); close(new_fd); connect_count--; //pthread_mutex_lock (&(curfds_lock)); //curfds--; //pthread_mutex_unlock (&(curfds_lock)); return NULL; } } /* 处理每个新连接上的数据收发结束 */ return NULL;}static void *handle_count(void* arg){ int precount, speed; while(1) { precount = msgcount; sleep(5); timecount += 5; //printf("The tcp connection count is %d\n",count_tcp); //printf("The received packets count is %d, time %d\n",msgcount, timecount); speed = msgcount - precount ; printf("The received speed is %d, connect %d\n",speed, connect_count); } return NULL;}int main(int argc, char **argv){ int listener, new_fd, nfds, n, ret; socklen_t len; struct sockaddr_in my_addr, their_addr; unsigned int myport, lisnum; struct epoll_event ev; struct rlimit rt; if(5 != argc) { printf("Usage: %s <thread_number(0 ~ 200)> <port(0-65535)> <listen queue number> <IP Address> \n", argv[0]); exit(1); } if(argv[1]) THREAD_NUMBER = atoi(argv[1]); if (argv[2]) myport = atoi(argv[2]); else myport = 7838; if (argv[3]) lisnum = atoi(argv[3]); else lisnum = 2; #ifdef DEBUG_TILERA if (tmc_cpus_get_my_affinity(&cpus) != 0) { printf("tmc_cpus_get_my_affinity() failed.\n"); tmc_task_die("tmc_cpus_get_my_affinity() failed."); } if (tmc_cpus_count(&cpus) < MAX_THREAD) { printf("\nInsufficient cpus available.\n"); tmc_task_die("Insufficient cpus available."); }#endif /*threadpool*/ pool_init (THREAD_NUMBER);/*线程池中最多THREAD_NUMBER个活动线程*/ pthread_mutex_init (&(curfds_lock), NULL); /* 设置每个进程允许打开的最大文件数 */ rt.rlim_max = rt.rlim_cur = MAXEPOLLSIZE; if (setrlimit(RLIMIT_NOFILE, &rt) == -1) { perror("setrlimit"); exit(1); } else printf("set the system resource success!\n"); /* 开启 socket 监听 */ if ((listener = socket(PF_INET, SOCK_STREAM, 0)) == -1) { perror("socket"); exit(1); } else printf("socket create success!n"); setnonblocking(listener); bzero(&my_addr, sizeof(my_addr)); my_addr.sin_family = PF_INET; my_addr.sin_port = htons(myport); if (argv[4]) my_addr.sin_addr.s_addr = inet_addr(argv[4]); else my_addr.sin_addr.s_addr = INADDR_ANY; if (bind (listener, (struct sockaddr *) &my_addr, sizeof(struct sockaddr)) == -1) { perror("bind"); exit(1); } else printf("IP address and port bing success!\n"); if (listen(listener, lisnum) == -1) { perror("listen"); exit(1); } else printf("start to work!\n"); /* 创建 epoll 句柄，把监听 socket 加入到 epoll 集合里 */ kdpfd = epoll_create(MAXEPOLLSIZE); len = sizeof(struct sockaddr_in); ev.events = EPOLLIN; ev.data.fd = listener; if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, listener, &ev) < 0) { fprintf(stderr, "epoll set insertion error: fd=%d\n", listener); return -1; } else printf("listen socket add to epoll success\n"); if (pthread_create(&thread_count, NULL, &handle_count, NULL) != 0) {#ifdef DEBUG_TILERA tmc_task_die("pthread_create() failed.");#endif } curfds = 1; while (1) { /* 等待有事件发生 */ nfds = epoll_wait(kdpfd, events, MAXEPOLLSIZE, -1); if (nfds == -1) { perror("epoll_wait"); break; } /* 处理所有事件 */ for (n = 0; n < nfds; ++n) { // printf("The number of fd %d \n", nfds); if (events[n].data.fd == listener) { new_fd = accept(listener, (struct sockaddr *) &their_addr, &len); if (new_fd < 0) { perror("accept"); continue; } //else // printf("connect from %x:%x， allocate socket for %x\n", inet_ntoa(their_addr.sin_addr), ntohs(their_addr.sin_port), new_fd); connect_count++; setnonblocking(new_fd); ev.events = EPOLLIN | EPOLLET; ev.data.fd = new_fd; if (epoll_ctl(kdpfd, EPOLL_CTL_ADD, new_fd, &ev) < 0) { fprintf(stderr, "add socket '%d' to epoll fail %s\n", new_fd, strerror(errno)); pool_destroy (); return -1; } //pthread_mutex_lock (&(curfds_lock)); // curfds++; // pthread_mutex_unlock (&(curfds_lock)); } else if(events[n].events & EPOLLIN ) { //printf("socket number %d", events[n].data.fd); pool_add_worker (handle_message, events[n].data.fd); //ret = handle_message(events[n].data.fd); /* if (ret < 1 && errno != 11) { epoll_ctl(kdpfd, EPOLL_CTL_DEL, events[n].data.fd, &ev); curfds--; }*/ /* ev.data.fd=events[n].data.fd; ev.events=EPOLLIN|EPOLLET; epoll_ctl(kdpfd,EPOLL_CTL_MOD, events[n].data.fd,&ev); */ } else { printf("other event \n"); } } } close(listener); pool_destroy (); return 0;}
