linux 下实现简单的异步多线程任务分发

linux 下实现简单的异步多线程任务分发

异步多任务并行处理在linux场景下有很多种实现方式。
当前主流有几种方案：多进程方案，1 master+N workers 方式，类似apache等等；多线程方案，或维护一个线程池。
本例子实现一个简单的单进程多线程异步分发任务模型。本实现有如下特点：
* 维护线程简单
* 直接可以传送函数，函数的参数
* 线程之间使用消息队列通信，实时性高，简单
* 可以实现简单的与主线程分离的任务
* swoole中，nodejs中的异步任务，需要在特定的io或事件场合，才可以异步，并不是随时可以异步，本例子实现随时任何情形都可以异步
受限于消息队列等因素，本例子还有未实现功能：
* 线程间没有锁机制，共享数据将不安全
* linux消息队列每个元素最大只能65535字节
* 暂时不能实现协程调度，工作线程和主线程是异步的，但是1个工作线程同时只能接受到1个任务，必须执行完才能接下一个任务。如果总共三个工作线程，每一个都阻塞，那就不能继续执行新的任务。
后续更新：
* 使用getcontext,setcontext,swapcontext来实现协程调度
* 实现有锁调用和无锁调用两种方式

code.c:

/* * 实现简单消息队列分发任务异步执行 * c99 thread.c -o thread -l pthread -D _XOPEN_SOURCE * */#define _XOPEN_SOURCE 1#include <pthread.h>#include <stdio.h>#include <stdlib.h>#include <unistd.h>#include <time.h>#include <sys/ipc.h>#include <sys/msg.h>#include <stdarg.h>#include <string.h>#define LOGF(STR, ...) logfmt(STR, __FILE__,__LINE__,__func__,##__VA_ARGS__)int msgqid = 0;             // 用于主线程和worker线程通信的消息队列id，由主线程申请int msgtype = 9812;         // 消息标识符，类型int IPC_KEY = 100011;       // 申请消息的IPC_KEY 本应该用ftok获得/* 消息队列消息结构 *//* 以下消息队列中保存的指针，只在当前运行进程内有效，其他进程或者重启进程将失效 */typedef struct message {    long msg_type;              /* 消息标识符 */    void* (*func)(void*);       /* 调用函数指针 在不同进程和进程重启后将失效 切记 */    void* param;                /* 此函数的参数 */    void** retval;              /* 此函数的返回值接收地址 发送方提供 发送方先申请好空间 */    void* (*callback)(void* retval, void*origin_param);   /* 完成后的callback函数 参数依次是 返回值，上次调用的函数的参数 */} message;/* 线程任务单元 */typedef struct threadtask {    pthread_t tid;    int is_working;    int working_times;} threadtask;/* 小线程池 */typedef struct taskpool {     threadtask* pool;     int len;} taskpool;taskpool* mainpool;void logfmt(const char* str, const char* file, const int line, const char* func, ...);/*protected 线程的main函数 task consumer */void* taskconsumer(void*ptr) {    threadtask* myth = (threadtask*)ptr;    struct message msgq;    int i = 0;    for (i=0; ;i++) {        msgrcv(msgqid, &msgq, sizeof(message)-sizeof(long), msgtype, 0);        myth->is_working = 1;        myth->working_times++;        LOGF("threadtask(%ld) got a task: loopid=%d taskfunc=%p param=%d\n", myth->tid, i, msgq.func, (msgq.param==NULL?0:*(int*)msgq.param));        if (msgq.func == NULL) {            LOGF("msgq.func is NULL, ignore this msg\n");            continue;        }        void*ret = msgq.func((void*)msgq.param);        if (msgq.retval != NULL) {            *(msgq.retval) = ret;        }        myth->is_working = 0;        if (msgq.callback != NULL) {            LOGF("this task has callback loopid=%d\n", i);            msgq.callback((void*)ret, msgq.param);        }    }}/*public user task*/void* task1(void* ptr) {    LOGF("task1 start, void*=%p v=%d\n", ptr, *(int*)ptr);    sleep(1);    LOGF("task1 over\n");    return NULL;}void* task2(void* ptr) {    LOGF("task2 start, void*=%x v=%d\n", ptr, *(int*)ptr);    sleep(10);    LOGF("task2 over\n");}void* callback1(void* ret, void* param) {    LOGF("i am callback: ret=%p param=%p\n", ret, param);}int recycletaskpool(taskpool* thepool);/*for main 分配线程池 init workers, waiting msgqueue for task*/taskpool* inittaskpool(int worker_num) {    taskpool* mainpool = (taskpool*)malloc(sizeof(taskpool));    memset(mainpool, 0, sizeof(*mainpool));    mainpool->pool = (threadtask*)malloc(sizeof(threadtask)*worker_num);    memset(mainpool->pool, 0, sizeof(*mainpool->pool));    mainpool->len = worker_num;    int has_error = 0;    int i = 0;    for (i=0; i<worker_num; ++i) {        pthread_t * tid_ptr = &(mainpool->pool[i].tid);        int ret = pthread_create(tid_ptr, NULL, (void *)taskconsumer, (void*)&(mainpool->pool[i]));        if (ret != 0) {            perror("create thread failed:");            has_error = 1;            break;        }        pthread_detach(*tid_ptr);    }    if (has_error) {        LOGF("create pthread error, go to recyclepool\n");        recycletaskpool(mainpool);    }    return mainpool;}/*for main 回收线程池资源*/int recycletaskpool(taskpool* thepool) {    if (thepool != NULL) {        if (thepool->pool != NULL) {            free(thepool->pool);        }        free(thepool);    }    return 0;}/*public 异步启动一个任务*/int async_do(void*(*func)(void*), void* param, void** ret, void* (*callback)(void*,void*)) {    struct message msgq;    msgq.msg_type = msgtype;    msgq.func = func;    msgq.param = param;    msgq.retval = ret;    msgq.callback = callback;    // send msgq to queue    int sendret = msgsnd(msgqid, &msgq, sizeof(message)-sizeof(long), 0);    return sendret;}int main() {    // 初始化队列    msgqid = msgget(IPC_KEY, IPC_CREAT | 0666);    // 初始化4个工作线程    int worker_num = 4;    mainpool = inittaskpool(worker_num);    char s[100] = {};    int sendret = 0;    void* (*taskfunc) (void*) = NULL;    void* (*thecallback) (void*,void*) = NULL;    int dorecycle = 0;    int i = 0;    for (i=0; i<16; i++) {        LOGF("input which task run: i=%d\n", i);        scanf("%s", s);        switch (s[0]) {            case '1':                taskfunc = task1;                thecallback = NULL;                break;            case '2':                taskfunc = task2;                thecallback = NULL;                break;            case '3':case '4':case '5':case '6':case '7':case '8':                taskfunc = task2;                thecallback = callback1;                break;            case 'q':                dorecycle = 1; // 退出                break;            default:                continue;                break;        }        if (dorecycle) {            break;        }        sendret = async_do(taskfunc, &s[0], NULL, thecallback);        //LOGF("you put v=%d when i=%d ret=%d\n", v, i, sendret);    }    // do recycle    int rmmsgret = msgctl(msgqid, IPC_RMID, NULL);    LOGF("msgqid %d have removed: ret=%d\n", msgqid, rmmsgret);    recycletaskpool(mainpool);    LOGF("mainpool has been recycled\n");}// logvoid logfmt(const char* str, const char* file, const int line, const char* func, ...) {    time_t now = time(NULL);    struct tm* tn = localtime(&now);    static char prestr[256] = {};    // 先预备好前缀    sprintf(prestr, "[%04d-%02d-%02d %02d:%02d:%02d] [%s:%d %s()] %s\x00", tn->tm_year+1900,tn->tm_mon+1,tn->tm_mday,tn->tm_hour,tn->tm_min,tn->tm_sec, file, line, func, str);    va_list ap;    va_start(ap, func);    // 不能使用printf 要使用vprintf    vprintf(prestr, ap);    va_end(ap);}

运行结果：

[root@iZ25gcs79rvZ thread]# ./thread [2017-07-13 16:54:44] [thread.c:164 main()] input which task run: i=01[2017-07-13 16:54:46] [thread.c:164 main()] input which task run: i=1[2017-07-13 16:54:46] [thread.c:57 taskconsumer()] threadtask(139760144594688) got a task: loopid=0 taskfunc=0x400bc9 param=49[2017-07-13 16:54:46] [thread.c:78 task1()] task1 start, void*=0x7fff4f3fb0b0 v=492[2017-07-13 16:54:46] [thread.c:164 main()] input which task run: i=2[2017-07-13 16:54:46] [thread.c:57 taskconsumer()] threadtask(139760152987392) got a task: loopid=0 taskfunc=0x400c32 param=50[2017-07-13 16:54:46] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=502[2017-07-13 16:54:47] [thread.c:164 main()] input which task run: i=3[2017-07-13 16:54:47] [thread.c:57 taskconsumer()] threadtask(139760161380096) got a task: loopid=0 taskfunc=0x400c32 param=50[2017-07-13 16:54:47] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=502[2017-07-13 16:54:47] [thread.c:80 task1()] task1 over[2017-07-13 16:54:47] [thread.c:164 main()] input which task run: i=4[2017-07-13 16:54:47] [thread.c:57 taskconsumer()] threadtask(139760169772800) got a task: loopid=0 taskfunc=0x400c32 param=50[2017-07-13 16:54:47] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=502[2017-07-13 16:54:48] [thread.c:164 main()] input which task run: i=5[2017-07-13 16:54:48] [thread.c:57 taskconsumer()] threadtask(139760144594688) got a task: loopid=1 taskfunc=0x400c32 param=50[2017-07-13 16:54:48] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=502[2017-07-13 16:54:48] [thread.c:164 main()] input which task run: i=6[2017-07-13 16:54:56] [thread.c:87 task2()] task2 over[2017-07-13 16:54:56] [thread.c:57 taskconsumer()] threadtask(139760152987392) got a task: loopid=1 taskfunc=0x400c32 param=50[2017-07-13 16:54:56] [thread.c:85 task2()] task2 start, void*=4f3fb0b0 v=50[2017-07-13 16:54:57] [thread.c:87 task2()] task2 over[2017-07-13 16:54:57] [thread.c:87 task2()] task2 over[2017-07-13 16:54:58] [thread.c:87 task2()] task2 overq[2017-07-13 16:55:04] [thread.c:196 main()] msgqid 917504 have removed: ret=0[2017-07-13 16:55:04] [thread.c:198 main()] mainpool has been recycled