网络爬虫中高效率的URL索引 更新支持千万条数据

网络爬虫中高效率的URL索引 

http://blog.csdn.net/chinafe/article/details/7816878

中使用数组方式 保存，但数组是有限制的，这里改进一下方式，使用向量 vector来实现 ，实现千万条数据没问题，测试时一千万的索引文件为9m

完整实现代码如下没

#include <stdio.h>#include <windows.h>#include <vector>using std::vector;void main(){vector<bool> vints(10000000, bool(0));//创建一个包含10000000个bool类型数据的vector，并且都初始化为0：DWORD dw_len=vints.size()*sizeof(bool);printf("write vints len:%d\n",dw_len);vints.at(10)=1;//随便写一条数据DWORD dwWrite;HANDLE hFileWrite=CreateFile("c:\\index.txt",GENERIC_WRITE,FILE_SHARE_READ,NULL,OPEN_ALWAYS,0,0);//保存索引文件if(hFileWrite==INVALID_HANDLE_VALUE) {CloseHandle(hFileWrite);return ;}WriteFile(hFileWrite,(char*)vints.begin(),dw_len,&dwWrite,0);CloseHandle(hFileWrite);vints.clear();vector<bool> vints_read(10000000, bool(0));//创建一个包含10000000个bool类型数据的vector，并且都初始化为0：HANDLE hFileRead=CreateFile("c:\\index.txt",GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING,0,0);////读取索引文件if(hFileRead==INVALID_HANDLE_VALUE)return;DWORD ret=0;if(!ReadFile(hFileRead,(char*)vints_read.begin(),dw_len,&ret,0)){MessageBox(0,"read","read index error",MB_OK);return;}CloseHandle(hFileRead);if (ret!=dw_len){MessageBox(0,"read","read index error",MB_OK);return;}printf("read  vints len:%d\n",ret);if (vints_read.at(10)==1)//测试读取的结果{printf("sucess\n");}return ;}