larbin中sequencer()及其相关函数阅读

<!--@page { margin: 2cm }P { margin-bottom: 0.21cm }-->

函数调用关系：

sequencer()

---canGetUrl()

------get()

------tryget()

------putPriorityUrlWait (url *u) { putGenericUrl(u, 1, true); }

------putPriorityUrl (url *u) { putGenericUrl(u, 5, true); }

------putUrlWait (url *u) { putGenericUrl(u, 10, false); }

------putUrl (url *u) { putGenericUrl(u, 15, false); }

/*start the sequencer*/void sequencer () { bool testPriority = true; if (space == 0) { space = global::inter->putAll();//能够装载的urls数量 } int still = space; if (still > maxPerCall) still = maxPerCall;//限定每次处理的最大值 while (still) {//还没处理完 if (canGetUrl(&testPriority)) { space--; still--;//要处理的urls减一，能装载的urls减一 } else { still = 0; }//end if (canGetUrl(&testPriority)) }//end while (still)}/*putPriorityUrlWait()，putPriorityUrl()，putUrlWait()，putUrl() *对于每个站点，将相应的URL放进去,最终调用的都是putGenericUrl*//*uint hostHashCode ();返回url的主机的哈西值*//*get();获得第一个对象*//*tryget();获得第一个对象,如果没有则返回NULL*//* Get the next url * here is defined how priorities are handled*/ static bool canGetUrl (bool *testPriority) { url *u; if (global::readPriorityWait) {//readPriorityWait是URLsPriorityWait中url的数量 global::readPriorityWait--; u = global::URLsPriorityWait->get(); global::namedSiteList[u->hostHashCode()].putPriorityUrlWait(u); return true; } else if (*testPriority && (u=global::URLsPriority->tryGet()) != NULL) { // We've got one url (priority) global::namedSiteList[u->hostHashCode()].putPriorityUrl(u); return true; } else { *testPriority = false; // Try to get an ordinary url if (global::readWait) {//readWait是URLsDiskWait中的url数量 global::readWait--; u = global::URLsDiskWait->get(); global::namedSiteList[u->hostHashCode()].putUrlWait(u); return true; } else { u = global::URLsDisk->tryGet();//获得第一个对象，如果没有返回空 if (u != NULL) { global::namedSiteList[u->hostHashCode()].putUrl(u); return true; } else { return false; }//end if (u != NULL) }//end if (global::readWait) }//end if (global::readPriorityWait)}/* 如果已经有足够多的url在内存里，执行if中的代码， * strcmp(name,u->getHost()) 是判断这个主机是不是已经就进行过dns方面的判断，也就是说对于一个站点，只做一次dns解析的判断，以后就按这个结果进行处理， * dnsState有noDns，noConnDns，还有robots.txt不允许的情况，如果没有问题，就把它放到URLsDisk中。*//* Put an url in the fifo if their are not too many */void NamedSite::putGenericUrl(url *u, int limit, bool prio) { if (nburls > maxUrlsBySite-limit) {//nburls为内存中的url数量，Already enough Urls in memory for this Site if (!strcmp(name, u->getHost())) {// first check if it can already be forgotten if (dnsState == errorDns) {//dns错误 nburls++; forgetUrl(u, noDNS); return; } if (dnsState == noConnDns) {//连接错误 nburls++; forgetUrl(u, noConnection); return; } if (u->getPort() == port && dnsState == doneDns && !testRobots(u->getFile())) {//不允许访问的内容 nburls++; forgetUrl(u, forbiddenRobots); return; } }//end if(!strcmp(name, u->getHost())) /*其他情况放到URLsDisk队列中*/ refUrl();// missUrl++ ; Number of urls reput in ram global::inter->getOne(); if (prio) {//优先性 global::URLsPriorityWait->put(u); } else { global::URLsDiskWait->put(u); } } /*如果需要判断dns能不能解析，就将它放到dnsSites里，这个会在fetchDns中判断。* 或是如果还能放到内存里，并且又是doneDns，表示可以解析，就调用transfer()；将url放入到IPSiteList的相应ipHash中*/ else { nburls++; if (dnsState == waitDns || strcmp(name, u->getHost()) || port != u->getPort() || global::now > dnsTimeout) { // dns not done or other site putInFifo(u); addNamedUrl(); // Put Site in fifo if not yet in if (!isInFifo) { isInFifo = true; global::dnsSites->put(this); } } else switch (dnsState) { case doneDns: transfer(u); /*将url放入到IPSiteList的相应ipHash中*/ break; case errorDns: forgetUrl(u, noDNS); break; default: // noConnDns forgetUrl(u, noConnection); } }//end if (nburls > maxUrlsBySite-limit)}