filemarge

 #include "filemarge.h"CFileMarge::CFileMarge(){ memset(this, 0, sizeof(*this));}CFileMarge::~CFileMarge(){}BOOL CFileMarge::Init(LPCSTR strFileA, LPCSTR strFileB, UINT nBufLen){ if (nBufLen > g_nMaxBufLen) { return FALSE; } m_pBuf = new char[nBufLen]; if (m_pBuf == NULL) { return FALSE; } m_pFileA = fopen(strFileA, "r"); if (m_pFileA == NULL) { return FALSE; } m_pFileB = fopen(strFileB, "r"); if (m_pFileB == NULL) { fclose(m_pFileA); return FALSE; } fclose(m_pFileA); fclose(m_pFileB); m_strFileA = strFileA; m_strFileB = strFileB; m_nBufLen = nBufLen; SetSegmentRange(); char szDir[255]; char szFileName[300]; getcwd(szDir, sizeof(szDir)); for (UINT m = 0; m < g_nThreadNum; m++) { m_szWorkThread[m] = new CWorkThread; m_szWorkThread[m]->Init(this, m); } for (UINT i = 0; i < g_nSmallFileNum; i++) { m_szSmallFileA[i] = new CWriteFile; sprintf(szFileName, "%s/a%02u", szDir, i); char* pBufPos = m_pBuf + ((nBufLen / 2) / g_nSmallFileNum) * i; m_szSmallFileA[i]->Init(szFileName, pBufPos, (nBufLen / 2) / g_nSmallFileNum , 30); m_szSmallFileB[i] = new CWriteFile; sprintf(szFileName, "%s/b%02u", szDir, i); pBufPos = pBufPos + nBufLen / 2; m_szSmallFileB[i]->Init(szFileName, pBufPos, (nBufLen / 2) / g_nSmallFileNum, 30); } return TRUE;}void CFileMarge::SetSegmentRange(){ m_pFileA = fopen(m_strFileA, "r"); m_pFileB = fopen(m_strFileB, "r"); fseek(m_pFileA, 0, SEEK_END); fpos_t nLenA; fgetpos(m_pFileA, &nLenA); UINT nPosB = 0; UINT nPosE = nLenA / (g_nThreadNum / 2); fpos_t nPos; fseek(m_pFileA, nPosE, SEEK_SET); for (UINT i = 0; i < g_nThreadNum / 2; i++) { char c; while ((c = fgetc(m_pFileA)) != '/n' && feof(m_pFileA) == 0); fgetpos(m_pFileA, &nPos); m_szSegmentRange[i][0] = nPosB; m_szSegmentRange[i][1] = nPos - 1; }}CWorkThread::CWorkThread(){ memset(this, 0, sizeof(*this));}CWorkThread::~CWorkThread(){}BOOL CWorkThread::Init(CFileMarge* pObj, UINT nDomain){ if (nDomain > g_nThreadNum) { return FALSE; } DWORD nThreadID; HANDLE hThread = CreateThread(NULL, 0, Func, (LPVOID)this, CREATE_SUSPENDED, &nThreadID); if (hThread == NULL) { return FALSE; } m_pFileManager = pObj; m_nDomain = nDomain; m_pFileManager->m_szhThread[nDomain] = hThread; return TRUE;}void CWorkThread::Destory(){ CloseHandle(m_pFileManager->m_szhThread[m_nDomain]);}void CWorkThread::Run(){ ResumeThread(m_pFileManager->m_szhThread[m_nDomain]);}void CWorkThread::CallBackSplitFile(){ CWriteFile** pWriteFile = m_nDomain < (g_nThreadNum / 2) ? m_pFileManager->m_szSmallFileA : m_pFileManager->m_szSmallFileB; FILE* file = fopen(m_pFileManager->m_strFileA, "r");}void CWorkThread::CallBackAnalyseNumber(){}DWORD CWorkThread::Func(LPVOID pa){ CWorkThread* pWorkThread = (CWorkThread*)pa; CFileMarge* pFileMarge = pWorkThread->m_pFileManager; BOOL bSplit = FALSE; BOOL bAnalyse = FALSE; for (; !pFileMarge->m_bExit; ) { switch (pFileMarge->m_nTaskState) { case TASK_STATE_SPLIT_FILE: { if (!bSplit) { pWorkThread->CallBackSplitFile(); bSplit = TRUE; } break; } case TASK_STATE_ANALYSE_NUMBER: { if (!bAnalyse) { pWorkThread->CallBackAnalyseNumber(); bAnalyse = TRUE; } break; } default: { Sleep(100); } } } return 0;}CWriteFile::CWriteFile(){ m_pFile = NULL; m_pFileBuf = NULL; m_nBufPos = 0; m_nBufLen = 0; m_nRecordLen = 0; m_nRecordCount = 0;}CWriteFile::~CWriteFile(){}BOOL CWriteFile::Init(LPCSTR strFileName, LPSTR pFileBuf, UINT nBufLen, UINT nRecordLen){ if (pFileBuf == NULL || nBufLen == 0 || nRecordLen == 0 || nBufLen < nRecordLen) { return FALSE; } m_pFile = fopen(strFileName, "w"); if (m_pFile == NULL || pFileBuf == NULL || nBufLen == 0 || nBufLen <= nRecordLen) { return FALSE; } m_pFileBuf = pFileBuf; m_nBufLen = nBufLen; m_nRecordLen = nRecordLen; return TRUE;}void CWriteFile::Destory(){ SaveFile(TRUE); fclose(m_pFile);}void CWriteFile::Append(LPCSTR strFormat, ...){ UINT nFormatLen = 0; va_list parameter; va_start(parameter, strFormat); AUTOLOCKBEGIN(m_Lock); m_nBufPos += vsprintf(m_pFileBuf + m_nBufPos, strFormat, parameter); AddRecordCount(); SaveFile(); AUTOLOCKEND;}void CWriteFile::AddRecordCount() { m_nRecordCount++;} UINT CWriteFile::GetRecordCount() const { return m_nRecordCount;} void CWriteFile::SaveFile(BOOL bForce){ if (m_nBufPos == 0) { return; } if (m_nBufLen - m_nBufPos < m_nRecordLen || bForce) { fwrite(m_pFileBuf, sizeof(CHAR), m_nBufPos, m_pFile); m_nBufPos = 0; }}

 

#ifndef FILE_MARGE_H#define FILE_MARGE_H#include <iostream>#include <direct.h>#include <stdarg.h>#include <windows.h>using namespace std;class CAutoLock{public: CAutoLock(CRITICAL_SECTION& obj) : m_pLockObj(&obj) { InitializeCriticalSection(m_pLockObj); EnterCriticalSection(m_pLockObj); } ~CAutoLock() { LeaveCriticalSection(m_pLockObj); DeleteCriticalSection(m_pLockObj); }private: CRITICAL_SECTION* m_pLockObj;};#define AUTOLOCKBEGIN(lock) { CAutoLock Lock ## __LINE__ (lock); #define AUTOLOCKEND } class CFileMarge;class CWorkThread;class CWriteFile;enum{ TASK_STATE_SPLIT_FILE = 0, TASK_STATE_ANALYSE_NUMBER};const UINT g_nThreadNum = 10;const UINT g_nSmallFileNum = 100;const UINT g_nMaxBufLen = 1024 * 1024 * 150;class CFileMarge{friend class CWorkThread;public: CFileMarge(); ~CFileMarge();public: BOOL Init(LPCSTR strFileA, LPCSTR strFileB, UINT nBufLen); void Destory(); void Run();private: void SetSegmentRange();private: BOOL m_bExit; LPCTSTR m_strFileA; LPCTSTR m_strFileB; LPCTSTR m_strFileC; UINT m_nMaskA; UINT m_nMaskB; CWorkThread* m_szWorkThread[g_nThreadNum]; UINT m_szSegmentRange[g_nThreadNum][2]; HANDLE m_szhThread[g_nThreadNum]; HANDLE m_szhEvent[g_nThreadNum]; FILE* m_pFileA; FILE* m_pFileB; char* m_pBuf; UINT m_nBufLen; CWriteFile* m_ResultFile; CWriteFile* m_szSmallFileA[g_nSmallFileNum]; CWriteFile* m_szSmallFileB[g_nSmallFileNum]; UINT m_nTaskState;};class CWorkThread{public: CWorkThread(); ~CWorkThread();public: BOOL Init(CFileMarge* pObj, UINT nDomain); void Destory(); void Run();private: void CallBackSplitFile(); void CallBackAnalyseNumber(); static DWORD WINAPI Func(LPVOID pa);private: UINT m_nDomain; HANDLE m_hThread; HANDLE m_hEvent; CFileMarge* m_pFileManager; };class CWriteFile{public: CWriteFile(); ~CWriteFile();public: BOOL Init(LPCSTR strFileName, LPSTR pFileBuf, UINT nBufLen, UINT nRecordLen); void Destory(); void Append(LPCSTR strFormat, ...); UINT GetRecordCount() const; private: void SaveFile(BOOL bForce = FALSE); void AddRecordCount(); private: FILE* m_pFile; LPCSTR m_strFileName; LPSTR m_pFileBuf; UINT m_nBufLen; UINT m_nBufPos; UINT m_nRecordLen; UINT m_nRecordCount; CRITICAL_SECTION m_Lock;};#endif

 

#include <stdio.h>#include <stdlib.h>#include <direct.h>#include <windows.h>#include "filemarge.h"const CHAR fileA[] = "f:/a.txt";const CHAR fileB[] = "f:/b.txt";const UINT nTelNum = 10000 * 10000;const UINT nWriteBuf = 1024 * 1024 * 64;const UINT nSingleRecordLen = 31;int main(){// mkdir("//tmp");// rmdir("//tmp"); CFileMarge a; a.Init(fileA, fileB, g_nMaxBufLen);// CWriteFile* a = new CWriteFile();// // if (a->Init("f:/nn.txt", new char[100], 100, 50))// {// char aa[] = "sunjian";// a->Append("%012u%s%u%s%u%s", 1234, aa, 1234, aa, 1234, aa);// a->Destory();// }// // delete a; return 0;// CHAR* pABuf = new CHAR[nWriteBuf];// // if (pABuf == NULL)// {// return 1;// }// // CHAR* pBBuf = new CHAR[nWriteBuf];// // if (pBBuf == NULL)// {// delete[] pABuf;// return 1;// }// // FILE* pFileA = fopen(fileA, "w");// // if (pFileA == NULL)// {// return 1;// }// // FILE* pFileB = fopen(fileB, "w");// // if (pFileB == NULL)// {// fclose(pFileA);// return 1;// }// // UINT nAPos = 0;// UINT nBPos = 0;// // for (UINT i = 0; i < nTelNum / 1000; i++)// {// nAPos += sprintf(pABuf + nAPos, "8613%09u,%u/n", i, i);// nBPos += sprintf(pBBuf + nBPos, "8613%09u,%u/n", i, i * 2);// // if (nWriteBuf - nAPos < nSingleRecordLen)// {// fwrite(pABuf, sizeof(CHAR), nAPos, pFileA);// nAPos = 0;// }// // if (nWriteBuf - nBPos < nSingleRecordLen)// {// fwrite(pBBuf, sizeof(CHAR), nBPos, pFileB);// nBPos = 0;// }// }// // if (nAPos > 0)// {// fwrite(pABuf, sizeof(CHAR), nAPos, pFileA);// nAPos = 0;// }// // if (nBPos > 0)// {// fwrite(pBBuf, sizeof(CHAR), nBPos, pFileB);// nBPos = 0;// }// // delete[] pABuf;// delete[] pBBuf;// // fclose(pFileA);// fclose(pFileB);// return 0;}