多线程之日志与消息分离读写
来源：互联网 发布：淘宝直播关注了在哪找 编辑：程序博客网 时间：2024/04/30 12:36
对于某些事件驱动行程序，往往存在大量的日志与消息的写入，如以单线程或者直接进行进程的写存会消耗进程大量的资源，不利于提高程效率。通多线程构建操作，将日写存交由线程处理，在日志写存量较大的程序中能显著的提高程序的运行效率。下面讨论的一种构将写日志的消息通过存取于队列之中，线程于队列中提消息进行写入。
线程相关知识不再这理作出相关讨论。百度上面有大量的相关知识。而我们首先要讨论的是Linux（centos 6.5）上面负责队列中相关消息收发函数。
int msgsnd(int msqid, const void *msgp, size_t msgsz, int msgflg);
msgsnd函数用于将到来的消息发送到队列中。其他各个形参对应的实际用途如下，
int msqid：为int msgget(key_t key, int msgflg);返回值 ，用于key创建的成功与否，在消息的收发之中，我们需要有一个固定的标识来标识不同的消息，如下存在3个消息队列。如无明确的标识，则会出现相关的混乱。
------ Message Queues --------
key msqid owner perms used-bytes messages
0x01050f47 262144 admin 666 0 0
0x01050f29 32769 admin 666 0 0
0x0104095c 65538 admin 600 3216 6
在实现msqid这前，我们需要定义实现key的赋值，可通过ftok实现。
key_t ftok(const char *pathname, int proj_id);
const void *msgp:发送至队列中相关消息缓存。
size_t msgsz：消息长度
int msgflg：相关标识
通过消息发送到队列中之后，我们则需要用msgrcv来进行读取队列中相关消息。
ssize_t msgrcv(int msqid, void *msgp, size_t msgsz, long msgtyp,int msgflg);
相关使用，于此不再作相关讨论。
在队列的结构体中，需要使用以下两个变量，long mtype 用于，如消息警告等级以及退出，可以通过定义mtpye来实现。
char mtext[]为发送消息缓存。
struct msgbuf {
long mtype; /* message type, must be > 0 */
char mtext[1]; /* message data */
};
msgctl用于控制消息队列 中消息的移除。可参考相关百度
int msgctl(int msqid, int cmd, struct msqid_ds *buf);
上面介绍了队列相关处理方法，下面介绍相关原理。
代码：
msg.h
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/ipc.h>
#include <sys/msg.h>
#define LOG_EXIT 0x0500+10 //退出
#define LOG_WRITE 0x0500+20 //写入
#define LOG_ID 30
#define LINE 4096
#define MSG_FILE "/dev/null"
//use for the line need to think about it carefully
////消息队列
struct msgbuf {
long mtype;
char mtext[2048];
};
msg.c 发送
#include "msg.h"
int msgsend(char *strSend,int type)
{
key_t key;
int nMsgid;
struct msgbuf data;
if((key = ftok(MSG_FILE, 1)) == -1){
perror("[error][ftok]ftok error");
return -1;
}
nMsgid = msgget(key+LOG_ID, 0666 | IPC_CREAT);
if(nMsgid == -1){
perror("[error][msgsend]msgget error");
return -1;
}
data.mtype = type ;
memset(data.mtext, 0, sizeof(data.mtext));
int nMaxSize = sizeof(data.mtext) - 1;
int nStrLen = strlen(strSend);
memcpy(data.mtext, strSend, nStrLen > nMaxSize ? nMaxSize : nStrLen);
if(msgsnd(nMsgid, (void*)&data, sizeof(data.mtext), IPC_NOWAIT) == -1){
perror("[error][msgsnd]msgsnd error");
return -1;
}
return 0;
}
write_log.h头文件
#include <dirent.h>
#include <time.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/msg.h>
#include <pthread.h>
#include <assert.h>
#define bool int
#define true 1
#define false 0
struct thread_info{
FILE* fd;
bool isopen;
pthread_t thread; //usr for the pthread as for pool of the pthread
char strPath[256];
int line;
// struct msgbuf data; //usr translated the data;
};
void print_log(const char* strFileName,int nLine,struct thread_info* pthread_data,const char *format,...);
void trace(const char* strFileName,int nLine, char* buf,struct thread_info* pthread_data);
bool write_log_open(char *strFileName,char * strMode,struct thread_info* pthread_data);
bool write_log(char* strContent,struct thread_info* pthread_data);
bool log_thread_stop(struct thread_info* pthread_data);
static void *log_create_thread(void* arg);
bool Log_write_string(const char *sFileName,int nFileLine,const char* strContent,char *end_string);
bool write_log_close(struct thread_info* pthread_data);
void pthread_init( struct thread_info* pthread_data);
write_log.c函数实现文件
#include "write_log.h"
#include "msg.h"
#define LOG_EXIT 0x0500+10 //退出
#define WRITE 0x0500+20 //写入
#define LOG_ID 30
#define BUF_LOG_LINE 4096
#define MSG_FILE "/dev/null"
//初始化掉结构体
void pthread_init( struct thread_info* pthread_data)
{
pthread_data->fd =NULL;
pthread_data->thread= NULL;
pthread_data->isopen = false;
memset(pthread_data->strPath,0,sizeof(pthread_data->strPath));
int line = 0;
}
bool write_log_close(struct thread_info* pthread_data)
{
printf("[fun] start to write_log_close...\n");
if(pthread_data->fd){
fflush(pthread_data->fd);
fclose(pthread_data->fd);
pthread_data->fd = NULL;
return true;
}
if(pthread_data->isopen == true){
pthread_data->isopen = false;
}
return false;
}
/**************************Log_write_string*********************/
/*用于在写入字符串中加入时间戳 *****/
/**************************Log_write_string*********************/
bool Log_write_string(const char *sFileName,int nFileLine,const char* strContent,char *end_string)
{
char str_title[LOG_TITLE+1] = {0};
char* strDest = NULL;
char* szTmpFileName = (char*)sFileName;
if(end_string == NULL){
end_string = (char*)malloc(2048);
}
strDest = (char*)malloc(strlen(strContent)+LOG_TITLE+200);
if(strDest == NULL || end_string == NULL){
return false;
}
int wMonth,wDay,wHour,wMinute,wSecond,wMilliseconds;
time_t timeval;
struct tm* tm_ptr;
struct timeval current;
gettimeofday(¤t, NULL);
timeval = current.tv_sec;
tm_ptr = localtime(&timeval);//or tm_ptr=gmtime(timeval);
wMonth = tm_ptr->tm_mon+1;
wDay = tm_ptr->tm_mday;
wHour = tm_ptr->tm_hour;
wMinute = tm_ptr->tm_min;
wSecond = tm_ptr->tm_sec;
wMilliseconds = current.tv_usec/1000;
sprintf(strDest,"%02d-%02d %02d:%02d:%02d.%03d(thread_id:0x%08X) %s,%s(%d)\n",
wMonth,wDay,wHour,
wMinute,wSecond,wMilliseconds,
(unsigned long)pthread_self(),strContent,
(char*)szTmpFileName,nFileLine);
strncpy(end_string,strDest,strlen(strDest));
//end_string = strDest;
// memcpy(end_string,strDest,sizeof(2048));
printf("[info][Log_write_string]will write to string is :%s\n",end_string);
return true;
}
/**************************log_create_thread*********************/
static void *log_create_thread(void *arg)
{
struct thread_info *thread_run_data = (struct thread_info *)arg;
printf("[fun][log_create_thread] log_create_thread start success,id is:0x%8x\n",pthread_self());
char* strBugLog = NULL;
char* eBugLog =(char *)malloc(2048);
strBugLog = (char *)malloc(2048);
if(strBugLog == NULL || eBugLog == NULL ){
sprintf(strBugLog,"[info]Log_thread(ThreadId:0x%08x) start.",
pthread_self());
write_log(strBugLog,thread_run_data);//start to the log
free(strBugLog);
strBugLog = NULL;
}
char* strFile = "netcat";
int msgid = -1;
int nFileLen = 0;
struct msgbuf data;
long msgtype = 0;
printf("[fun][log_create_thread] start success...\n");
key_t key = -1;
printf("[info][log_creat_thread] key value is :%d ,msgid value is：%d\n",key,msgid);
if((key = ftok(MSG_FILE, 1)) == -1) //系统建立IPC通讯（如消息队列、共享内存时）必须指定一个ID值
{
perror("[error][log_creat_thread] ftok error");
goto thread_log_exit;
}
msgid = msgget(key+LOG_ID, 0666 | IPC_CREAT);
if(msgid == -1){
perror("[error][log_creat_thread] msget error");
goto thread_log_exit;
}
printf("[info][log_write_thread] key value is :%d ,msgid value is：%d\n",key,msgid);
while(1){
printf("*************************************************\n");
system("ipcs -q");
printf("*************************************************\n");
int ret = msgrcv(msgid,&data,sizeof(data.mtext),msgtype,MSG_NOERROR);//MSGNOERROR
Log_write_string(thread_run_data->strPath,thread_run_data->line,data.mtext,strBugLog);
printf("*************************************************\n");
system("ipcs -q");
printf("*************************************************\n");
printf("strBugLog is %s\n",strBugLog);
if(ret == -1){
perror("[error][log_write_thread] msgrcv is failed");
goto thread_log_exit;
}
switch (data.mtype)
{
case WM_LOG_WRITE:
write_log(strBugLog,thread_run_data);
nFileLen += strlen(strBugLog);
if(nFileLen >= 1024*1024*50){
write_log_open(strFile,"w",thread_run_data);
nFileLen = 0;
}
break;
case WM_LOG_EXIT:
{
goto thread_log_exit;
}break;
default:
{
}break;
}
memset(strBugLog,0,sizeof(strBugLog));
}
thread_log_exit:
printf("[info][log_write_thread] error for thread_log_exit\n");
if(NULL != strBugLog){
free(strBugLog);
strBugLog = NULL;
}
if(eBugLog){
sprintf(eBugLog,"ThreadId:0x%08X) exit.",pthread_self());
msgctl(msgid,IPC_RMID,NULL);
write_log(eBugLog,thread_run_data);
free(eBugLog);
eBugLog = NULL;
write_log_close(thread_run_data->fd);
return 0;
}
}
/****************************log_thread_stop**********************/
/**关闭线程 ****/
/****************************log_thread_stop**********************/
bool log_thread_stop(struct thread_info *pthread_data){
printf("[fun][log_thread_stop]start success...\n");
char *strStop ="The thread will stoped";
char end_str[2048] ={0};
if(pthread_data->thread != NULL)
{
pthread_data->line +=1;
Log_write_string(pthread_data->strPath,pthread_data->line,strStop,end_str);
printf("[info][log_thread_stop] the pThread is send message to stop...\n");
msgsend(end_str,EXIT);
}
printf("[info][log_thread_stop] the pThread is not start...\n");
return true;
}
/****************************log_start_thread**********************/
bool log_start_thread(struct thread_info *pthread_data){
int s;
log_thread_stop(pthread_data);
s = pthread_create(&pthread_data->thread,NULL,&log_create_thread,pthread_data);
printf("[info][log_start_thread] pthread_data->fd :0x%8x\n",pthread_data->fd);
printf("[info][log_start_thread] pthread_data->thread :0x%8x\n",pthread_data->thread);
printf("[info][log_start_thread] pthread_data->isopen :%d\n", pthread_data->isopen);
printf("[info][log_start_thread] pthread_data->strPath :%s\n", pthread_data->strPath);
if(0 == s){
printf("[info] The thread start success..\n ");
return true;
}
return false;
}
/********************log_write*************************/
bool write_log(char* strContent,struct thread_info *pthread_data)
{
printf("[fun][write_log] write_log start success...%s\n",strContent);
assert(strContent && pthread_data);
if(pthread_data->isopen == true)
{
if(pthread_data->fd){
fwrite(strContent,sizeof(char),strlen(strContent),pthread_data->fd);
fflush(pthread_data->fd);
return true;
}
else
{
printf("[error][write_log]error for the open file\n");
return false;
}
}
printf("[error][write_log] not open the file for write\n");
return false;
}
/********************write_log_open**********************/
bool write_log_open(char *strFileName,char* strMode,struct thread_info *pthread_data)
{
time_t timeval;
struct tm* tm_ptr;
char strFileName_tmp[256] = {0};
int mMonth,mDay,mHour,mMinute,mSecond,mMilliseconds,mYear;
timeval = time(NULL);
tm_ptr = localtime(&timeval);
mMonth = tm_ptr->tm_mon;
mDay = tm_ptr->tm_mday;
mHour = tm_ptr->tm_hour;
mMinute = tm_ptr->tm_min;
mSecond = tm_ptr->tm_sec;
mYear = tm_ptr->tm_year;
sprintf(strFileName_tmp,"%s_%04d%02d%02d%02d%02d%02d.log",
strFileName,mYear,mMonth,mDay,mHour,mMinute,mSecond);
assert(strFileName != NULL && strMode !=NULL);
printf("[fun][write_log_open]strMod is :%s\n",strMode);
pthread_data->fd = fopen(strFileName_tmp,strMode);
if(NULL == pthread_data->fd){
printf("[error][write_log_open] open the %s file error\n",strFileName_tmp);
return false;
}
pthread_data->isopen = true;
printf("[fun] The write_log_open success...\n");
printf("[info] strFileName is :%s\n",strFileName_tmp);
return true;
}
/******************************* trace **********************/
/*e有无线程情况的判断及文件打开 */
/* */
/**************************************************************/
void trace(const char* strFileName,int nLine, char* buf,struct thread_info *pthread_data)
{
printf("[fun][trace] start success....\n");
char end_string[2047]={0};
if(pthread_data->thread == NULL)
{
char sFile[256] = {0};
char sPath[256] = {0};
char sLogPath[256] = {0};
int rslt = readlink("/proc/self/exe",sFile,256);
if(rslt > 0 || rslt <= 256){
sFile[rslt] = '\0';
}
char *pStr = strrchr(sFile,'/');
strncpy(sPath,sFile,pStr-sFile+1);
printf("[fun][trace]sPath is:%s\n",sPath);
char sFixedPath[256] = {0};
strcpy(sFixedPath,sPath);
strcat(sFixedPath,"log");
printf("[fun][trace]sFixedPath is:%s\n",sFixedPath);
DIR* dp = NULL;
dp = opendir(sFixedPath);
if(NULL == dp){
mkdir(sFixedPath,0x774);
strcat(sPath,"log/netcat_");
printf("[fun][trace]sPath is:%s\n",sPath);
}
else
{
strcat(sPath,"log/netcat_");
printf("[fun][trace]opendir success");
printf("[fun][trace]sPath is:%s\n",sPath);
}
strcat(sLogPath,sPath);
printf("Path:%s\n",sLogPath);
//先发消息进入队列，
msgsend(buf,WRITE);
system("ipcs -q");
//由结构体中的线程可知，线程未被创建
//此处为创建线程。先创建线程，
//先打开文件进行写操作
Log_write_string(pthread_data->strPath,pthread_data->line,buf,end_string);
write_log_open(sLogPath,"w+",pthread_data);
log_start_thread(pthread_data);
// pthread_join(pthread_data->thread,NULL);
}
Log_write_string(pthread_data->strPath,pthread_data->line,buf,end_string);
msgsend(end_string,WRITE);
// pthread_join(pthread_data->thread,NULL);
}
/*******************************trace fun**********************/
/*暂不加警告等级 */
/*c参数转换，后期等级加入 */
/**************************************************************/
void print_log(const char* strFileName,int nLine,struct thread_info* pthread_data,const char *format,...)
{
printf("[fun][print_log] start success....\n");
char buf[LINE];
va_list v1;
va_start(v1,format);
vsnprintf(buf,sizeof(buf),format,v1);
va_end(v1);
strncpy(pthread_data->strPath,strFileName,strlen(strFileName));
pthread_data->line = nLine;
trace(strFileName,nLine,buf,pthread_data);
}
- 多线程之日志与消息分离读写
- Java多线程之~~~ReadWriteLock 读写分离的多线程实现
- Java多线程之~~~ReadWriteLock 读写分离的多线程实现
- 日志系列之接口与实现分离
- 读写分离之Amoeba
- Nginx之读写分离
- mysql读写分离之代码分析与设计（2）
- Mycat之——读写分离与主从切换案例
- MySQL复制与读写分离
- Mycat读写分离与注意事项
- MySQL复制与读写分离
- mysql 读写分离 之 amoeba
- NoSQL之Redis读写分离
- mysql读写分离之amoeba
- 数据库优化之读写分离
- redis集群之读写分离
- 多线程之读写锁
- 多线程之读写锁
- 移动端开发
- 【Summary】——Web组·大三·Week2
- 13期 10月期刊自荐
- tensorflow更改变量的值
- leetcode322 coin change & leetcode343 integer break
- 多线程之日志与消息分离读写
- 2017 ACM-ICPC 亚洲区（南宁赛区）网络赛 B. Train Seats Reservation 和 F. Overlapping Rectangles
- 世上最全SpringBoot视频教程
- poj 3211 Washing Clothes
- GFS-Google论文阅读笔记
- 文章标题
- c++primer 第五版3.23节练习
- Python转义字符
- 在线预览功能的实现
