linux下http多线程下载案例

/**********************************/


//GET /linuxqq/linuxqq-v1.0.2-beta1.i386.rpm HTTP/1.1
//Accept: */*
//Accept-Language: en-us
//User-Agent:Mozilla/4.0 (compatible; MSIE 5.01; Window NT 5.0)
//Host: :80
//Connection: Keep-Alive
//HTTP/1.1 206 Partial Content
//Content-Length: 1009349
//Content-Range: bytes 0-1009348/5046743
//Server: qqdlsrv(1.84 for linux)
//Connection: Keep-alive
//Content-Disposition: attachment; filename=linuxqq-v1.0.2-beta1.i386.rpm
//Accept-Ranges:bytes
//Content-Type:applivation/octet-stream


#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netdb.h>
#include <netinet/in.h>




//#define PORT 80


//create a struct to save the file part infomation
typedef struct tagFilePart{
int partNum;
char parhName[128];
long int length;
long int dataStart;
long int dataEnd;
struct tagFilePart *pNext;
}FilePart;




struct hostent *pMyHttpHost;
char pMyFilePath[2048]={0}; // save the filepath on the server
char pMyHostName[1024]={0}; // save the name of the server
char pMyFileName[256]={0}; // save the name of the file
char pMyLocalPath[512]={0};
int tMyPort, tMyPart;
long int tMyFileLen;


void GetName(char *pInputUrl);
void SetFilePart(long int tInputDataLen, int tInputPart);
void *GetFile(FilePart *pInputPoint);
long int GetFileLen(struct hostent *pInputHost);
void CreateThread(FilePart *pInputHead, int tInputNum);
int MergeFile(FilePart *pInputHead);






//http://dl_dir.qq.com/linuxqq/linuxqq-v1.0.2-beta1.i386.rpm
///home/damon/test/linux_http
int main(int argc, char *argv[])
{
char pTempUrl[128]={0};


printf("please input the website: \n");
scanf("%s", pTempUrl);
printf("the website is : %s \n", pTempUrl);
printf("input the direct to save file : \n");
scanf("%s",  pMyLocalPath);
GetName(pTempUrl);// get the ipaddress of the host
tMyFileLen=GetFileLen(pMyHttpHost);// get the length of the file
SetFilePart(tMyFileLen, 3);


return 0;
}




long int GetFileLen(struct hostent *pInputHost)
{
struct sockaddr_in tTempSockAddr;
int tTempSockFd;
char pTempRequest[1024];
long int tTempFileLen;
int tTempRecvBytes;
char *pTempBuf, *pTempBufP, *pTempPt;


pTempBuf=(char *)malloc(1024*sizeof(char));
memset(pTempBuf, 0, sizeof(pTempBuf));
tTempSockFd=socket(AF_INET, SOCK_STREAM, 0);
if(tTempSockFd<0)
{
perror("socket");
exit(-1);
}


memset(&tTempSockAddr, 0, sizeof(tTempSockAddr));
tTempSockAddr.sin_family=AF_INET;
tTempSockAddr.sin_port=htons(tMyPort);
tTempSockAddr.sin_addr=*((struct in_addr *)pInputHost->h_addr);
if(0>connect(tTempSockFd, (struct sockaddr *)&tTempSockAddr, sizeof(struct sockaddr)))
{
perror("connect");
exit(-1);
}
printf("connect success !\n");
printf("file path is:%s\n", pMyFilePath);


sprintf(pTempRequest, "GET %s HTTP/1.1\r\nAccept: */*\r\nAccept-Language:en-us\r\nUser-Agent:Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\r\nHost: %s:%d\r\nConnection: Keep-Alive\r\n\r\n",
pMyFilePath, pMyHostName, tMyPort);
printf("%s", pTempRequest);
// send the http protocl packet
if(0>send(tTempSockFd, pTempRequest, strlen(pTempRequest), 0))
{
perror("send");
exit(1);
}


while(1)
{
memset(pTempBuf, 0, 1024);
tTempRecvBytes=recv(tTempSockFd, pTempBuf, 1024, 0);
printf("recvbytes=%d \n", tTempRecvBytes);
printf("damon===> pTempBuf=[%s]\n", pTempBuf);
if(0>tTempRecvBytes)
{
perror("recv");
exit(1);
}


pTempPt=strstr(pTempBuf, "Length");
if(NULL!=pTempPt)
{
pTempPt=strchr(pTempPt, ':');
pTempPt++;
tTempFileLen=atol(pTempPt);
printf("the file data is %ld \n", tTempFileLen);
}


pTempBufP=strstr(pTempBuf, "\r\n\r\n");
printf("pTempBufp=[%s]\n", pTempBufP);
if(NULL!=pTempBufP)
break;
}


free(pTempBuf);
pTempBuf=NULL;
close(tTempSockFd);

return tTempFileLen;
}




void GetName(char *pInputUrl)
{
char pTempHead[10]={0};// save the "http:// or "https://""
// char pTempHostName[1024]={0};
char *pTempPoint1;
char *pTempPoint2;
char *pTempAddr;


pTempPoint1=strstr(pInputUrl, "//");// move the pointer to the "//" of the url
pTempPoint1+=2;// move the pointer to the right of "//" of the url
strncpy(pTempHead, pInputUrl, pTempPoint1-pInputUrl);// copy "http://" to head
printf("head=[%s]\n", pTempHead);
pTempPoint2=strchr(pTempPoint1, ':');// find the port of the server
if(pTempPoint2==NULL)// if there is no port
{
char *pTempPt=strchr(pTempPoint1, '/');// move pointer to the first '/'
tMyPort=80;
if(pTempPt!=NULL)
{
pTempPoint2=pTempPt++;// save a potiner pointo the right of the first
strncpy(pMyHostName, pTempPoint1, pTempPoint2-pTempPoint1);// save the host name
strcpy(pMyFilePath, pTempPoint2);// the right of the first '/' are the filepath and name, then save !
printf("filePath: %s \n", pMyFilePath);
while(NULL!=pTempPt)
{
pTempPt=strchr(pTempPt, '/');// find the right '/'
if(NULL!=pTempPt)
{
pTempPoint2=pTempPt;
pTempPt++;
}
}


strcpy(pMyFileName, ++pTempPoint2); // the right of the right '/' is the file name !
printf("fileName is : %s \n", pMyFileName);
}else // not find the first '/' or pTemppt==NULL
{
strcpy(pMyHostName, pTempPoint1);
strcpy(pMyFilePath, "/index.html");
printf("filePath==[%s]\n", pMyFilePath);
}
}else
{
strncpy(pMyHostName, pTempPoint1, pTempPoint2-pTempPoint1);
tMyPort=atoi(++pTempPoint2);
printf("damon==> port=%d \n", tMyPort);
}
printf("hostName=[%s]\n", pMyHostName);
pMyHttpHost=gethostbyname(pMyHostName);
if(NULL==pMyHttpHost)
{
perror("gethostbyname");
exit(1);
}


// pTempAddr=pMyHttpHost->h_addr;//pMyHttpHost->h_addr_list[0];// char **addrs;
// printf("%s", pTempAddr);
// printf("\n");
}


void SetFilePart(long int tInputDataLen, int tInputPart)// move the file length and the number of cutting parts
{
int i;
long int tTempPartLen=tInputDataLen/tInputPart;
long int tTempEnd=tInputDataLen-1;
printf("partLen=%ld, end=%ld \n", tTempPartLen, tTempEnd);


FilePart *pTempHead, *pTempPresent;
pTempHead=(FilePart *)malloc(sizeof(FilePart));// create the head of the link
pTempHead->partNum=1;
pTempHead->dataStart=0;
pTempHead->dataEnd=tTempPartLen;
pTempHead->length=tTempPartLen;
sprintf(pTempHead->parhName, "%s/part%d", pMyLocalPath, 1);
if(tInputPart==1)
pTempHead->dataEnd=tTempEnd;
pTempPresent=pTempHead;
for(i=0; i<tInputPart-1; i++)
{
pTempPresent->pNext=(FilePart *)malloc(sizeof(FilePart));
memset(pTempPresent->pNext, 0, sizeof(FilePart));
pTempPresent->pNext->partNum=i+2;
pTempPresent->pNext->dataStart=pTempPresent->dataEnd+1;
if(i==(tInputPart-2))// judge if it is the last part !
{
pTempPresent->pNext->dataEnd=tTempEnd;
pTempPresent->pNext->length=tInputDataLen-(tInputPart-1)*tTempPartLen;
}else
{
pTempPresent->pNext->dataEnd=tTempPartLen*(i+2);
pTempPresent->pNext->length=tTempPartLen;
}
sprintf(pTempPresent->pNext->parhName, "%s/part%d", pMyLocalPath, (i+2));
pTempPresent=pTempPresent->pNext;
}


CreateThread(pTempHead, tInputPart);
// MergeFile(pTempHead);
}


void CreateThread(FilePart *pInputHead, int tInputNum)
{
printf("start to create thread \n");
int i=1, tTempRet;
long int pTempPres[tInputNum];
pthread_t pTempHandle[tInputNum];
FilePart *pTempPresent=pInputHead;




for(i=0; i<tInputNum; i++)
{
printf("damon===> %ld %ld %ld \n",pTempPresent->dataStart, pTempPresent->dataEnd, pTempPresent->length);
tTempRet=pthread_create(&pTempHandle[i], NULL, GetFile, pTempPresent);
if(tTempRet!=0)
{
perror("thread creation failed");
exit(1);
}
printf("start to create thread %d \n", i+1);
pTempPresent=pTempPresent->pNext;
}




long int tTempAll=0;
for(i=0; i<tInputNum; i++)
{
tTempRet=pthread_join(pTempHandle[i], &pTempPres[i]);
if(tTempRet!=0)
{
perror("pthread join");
exit(1);
}
printf("thread %d finished ! \n", i+1);
tTempAll += pTempPres[i];
}
printf("all download: %ld! \n", tTempAll);
}


void *GetFile(FilePart *pInputPoint)
{
char pTempBuffer[1024], pTempRequest[2048];
struct sockaddr_in tTempSockAddr;
FILE *pTempFp, *pTempFpLog;
int tTempRecvBytes, tTempSockFd, tTempSaveBytes;
long int tTempDownload=0;


tTempSockFd=socket(AF_INET, SOCK_STREAM, 0);
if(tTempSockFd<0)
{
printf("damon=====> socket failed \n");
perror("socket");
pthread_exit(1);
}


memset(&tTempSockAddr, 0, sizeof(tTempSockAddr));
tTempSockAddr.sin_family=AF_INET;
tTempSockAddr.sin_port=htons(tMyPort);
tTempSockAddr.sin_addr=*((struct in_addr *)pMyHttpHost->h_addr);
if((connect(tTempSockFd, (struct sockaddr *)&tTempSockAddr, sizeof(struct sockaddr)))<0)
{
printf("damon====+> connect failed \n");
perror("connect ");
pthread_exit(1);
}


sprintf(pTempRequest, "GET %s HTTP/1.1\r\nAccept: */*\r\nAccept-Language:en-us\r\nUser-Agent: Mozilla/4.0 (compatible;MSIE 5.01; Windows NT 5.0)\r\nHost: %s:%d\r\nRange:bytes=%ld-%ld\r\nConnection:Keep-Alive\r\n\r\n", 
pMyFilePath, pMyHostName, tMyPort, pInputPoint->dataStart, pInputPoint->dataEnd);
printf("damon==> request=[%s]\n", pTempRequest);
int tTempSend=send(tTempSockFd, pTempRequest, strlen(pTempRequest), 0);
if(tTempSend<0)
{
perror("send");
exit(1);
}


char pTempFileLog[64]={0};
sprintf(pTempFileLog, "/home/damon/test/linux_http/part%dlog", pInputPoint->partNum);
pTempFpLog=fopen(pTempFileLog, "w+");
pTempFp=fopen(pInputPoint->parhName, "w+");
if(pTempFp<0)
{
perror("open");
pthread_exit(1);
}


int tTempFlag=0;


do{
memset(pTempBuffer, 0, sizeof(pTempBuffer));
tTempRecvBytes=recv(tTempSockFd, pTempBuffer, sizeof(pTempBuffer), 0);
printf("damon====> recvByte=[%d], pTempBuffer=[%s]\n", tTempRecvBytes, pTempBuffer);
if(tTempRecvBytes==0)
break;
if(tTempRecvBytes<0)
{
perror("recv");
pthread_exit(1);
}


char *pTempPt;
pTempPt=strstr(pTempBuffer, "\r\n\r\n");


if(!tTempFlag)
{
if(pTempPt!=NULL)
{
pTempPt += 4;
tTempFlag=fwrite(pTempBuffer, 1, (pTempPt-pTempBuffer), pTempFpLog);
tTempSaveBytes=tTempRecvBytes-(pTempPt-pTempBuffer);
if(tTempSaveBytes>0)
tTempFlag=fwrite(pTempPt, 1, tTempSaveBytes, pTempFp);


tTempDownload+=tTempFlag;
}else
{
printf("tan==> pTempPt==NULL \n");
}
}else
{
tTempFlag=fwrite(pTempBuffer, 1, tTempRecvBytes, pTempFp);
tTempDownload+=tTempFlag;
}


tTempRecvBytes=0;
}while(tTempRecvBytes>0);


printf("%ld have downloaded !\n", tTempDownload);
fclose(pTempFp);
fclose(pTempFpLog);
close(tTempSockFd);


pthread_exit(tTempDownload);
}




//merge all parts to one file and remove the temp file rename the file
int MergeFile(FilePart * pInputHead)
{
FilePart *pTempPresent, *pTempRelease;
char *pTempBuffer=(char *)malloc(1024*sizeof(char));
FILE *pTempFread, *pTempFwrite;
int tTempWriteNum, tTempReadNum;


pTempFwrite=fopen(pInputHead->parhName, "ab");// open the first part as append mode
if(pTempFwrite==NULL)
{
printf(" open file %s failed !\n", pInputHead->parhName);
exit(1);
}


pTempPresent=pInputHead->pNext;
while(pTempPresent)
{
pTempFread=fopen(pTempPresent->parhName, "rb");// open temp part's data
if(pTempFread==NULL)
{
perror("fopen");
exit(1);
}else
{
printf("open success ! \n");
}


while(!feof(pTempFread))
{
tTempReadNum=fread(pTempBuffer, sizeof(char), 1024, pTempFread);
tTempWriteNum=fwrite(pTempBuffer, sizeof(char), tTempReadNum, pTempFwrite);
}
printf("read %d part \n", pTempPresent->partNum);
fclose(pTempFread);
remove(pTempPresent->parhName);
pTempRelease=pTempPresent;
pTempPresent=pTempPresent->pNext;
free(pTempRelease);
}


free(pTempBuffer);
pTempBuffer=NULL;
fclose(pTempFwrite);
sprintf(pMyLocalPath, "%s/%s", pMyLocalPath, pMyFileName);
if(0>(rename(pInputHead->parhName, pMyLocalPath)))
perror("rename ");
free(pInputHead);


return 0;
}