url normalization

在爬虫中需要将url变为唯一、合法的形式以避免重复爬行,因此正规化要将其中必须用urlencode的地方全部encode,不需要的即时在原始url中已经encode也要decode(保留字符和非保留字符使用原RFC规定的,参考维基百科url normalization一文,与浏览器有一定出入),且所有的url encode都要使用大写形式,域名后面的冗余点号必须去掉,域名必须全部小写,域名部分还要按照IDN的规定使用punycode进行编码(暂未实现),路径部分要消除.和..的冗余,同事对于超出范围的url要能容错(如:http://sports.163.com/./xj/../.. => http://sports.163.com/),query strings中的key-value对排序之后连接。对于空格统一使用+。由于逻辑判断较多,最后写成了有限状态机的形式,以使逻辑清晰。相当于有两层词法分析,第一层按照需要urlencode或者decode之后生成一些字节,第二层则以第一层输出为输入,进行normalize。其中的状态机终态不互为前缀,这样状态迁移的过程就不必保存上次匹配了。

值得商榷的地方是空格全部使用+,而这时早期的规定,不知道是否所有服务器都支持(这样实现略简单些)。另外输入的串必须以/0结束,也就是传入的未归一的url中不能包含0字节。不过目前还没有在中间有0字节的需求。如果把诸多的if给成switch经优化后可能会加快。

google url貌似也完成类似功能。这里贴出fileNormalize的代码。// !*'();:@&=+$,/?#[]// %21 %2A %27 %28 %29 %3B %3A %40 %26 %3D %2B %24 %2C %2F %3F %23 %5B %5Dconst int reserved[] = { /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 00,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 10,1,0,1, 1,0,1,1, 1,1,1,1, 1,0,0,1, // 20,0,0,0, 0,0,0,0, 0,0,1,1, 0,1,0,1, // 31,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 40,0,0,0, 0,0,0,0, 0,0,0,1, 0,1,0,0, // 50,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 60,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 70,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 80,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 90,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // A0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // B0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // C0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // D0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // E0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // F};// A-Za-z0-9-_.~// %41-%5A,%61-%7A,%30-%39,%2d,%5F,%2e,%7econst int unreserved[] = {/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 00,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 10,0,0,0, 0,0,0,0, 0,0,0,0, 0,1,1,0, // 21,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0, // 30,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, // 41,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,1, // 50,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, // 61,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,1,0, // 70,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 80,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // 90,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // A0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // B0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // C0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // D0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // E0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, // F};// return the int with correspond to a char// -1 if not an hexa charstatic int int_of_hexa (char c) { if (c >= '0' && c <= '9') return (c - '0'); else if (c >= 'a' && c <= 'f') return (c - 'a' + 10); else if (c >= 'A' && c <= 'F') return (c - 'A' + 10); else return -1;}// get the next encode stateinline void handle_url_encode_state(int& encode_state, char c, int& val, std::vector<char>& vec) { // to handle url encode and decode char buf[3]; switch (encode_state) { case 0: if (c == '%') { vec.push_back('%'); encode_state = 1;} else if (c == 0x20) {vec.push_back('+'); } else if (!reserved[(unsigned char)c] && !unreserved[(unsigned char)c] && c != 0) { vec.push_back('%'); sprintf(buf, "%X", ((unsigned char)c >> 4)); vec.push_back(buf[0]); sprintf(buf, "%X", ((unsigned char)c & 0x0f)); vec.push_back(buf[0]); } break; case 1: if (c == '%') { vec.push_back('2'); vec.push_back('5'); vec.push_back('%'); encode_state = 1; } else if (('0' <= c && c <= '9') || ('a' <= c && c <= 'f')|| ('A' <= c && c <= 'F')) { vec.push_back(c); encode_state = 2; val = int_of_hexa(c); } else if (c == 0x20) { vec.push_back('2'); vec.push_back('5'); vec.push_back('+'); encode_state = 0; } else if (!reserved[(unsigned char)c] && !unreserved[(unsigned char)c] && c != 0) {vec.push_back('2');vec.push_back('5'); sprintf(buf, "%X", ((unsigned char)c >> 4)); vec.push_back(buf[0]); sprintf(buf, "%X", ((unsigned char)c & 0x0f)); vec.push_back(buf[0]); encode_state = 0; } else { vec.push_back(c); encode_state = 0; } break; case 2: if (c == '%') { c = vec.back(); vec.pop_back(); vec.push_back('2'); vec.push_back('5'); vec.push_back(c); vec.push_back('%'); encode_state = 1; } else if (('0' <= c && c <= '9') || ('a' <= c && c <= 'f')|| ('A' <= c && c <= 'F')) { val *= 16; val += int_of_hexa(c); if (val == 0x20) { vec.pop_back(); vec.pop_back(); vec.push_back('+'); } else if (!unreserved[val] || c == 0) { vec.pop_back(); sprintf(buf, "%X", (val >> 4)); vec.push_back(buf[0]); sprintf(buf, "%X", (val & 0x0f)); vec.push_back(buf[0]); } else { vec.pop_back(); vec.pop_back(); vec.push_back((char)val); } encode_state = 0; } else if (c == 0x20) { c = vec.back(); vec.pop_back(); vec.push_back('2'); vec.push_back('5'); vec.push_back(c); vec.push_back('+'); encode_state = 0; } else if (!reserved[(unsigned char)c] && !unreserved[(unsigned char)c] && c != 0) { sprintf(buf, "%X", ((unsigned char)c >> 4)); sprintf(buf + 1, "%X", ((unsigned char)c & 0x0f)); c = vec.back(); vec.pop_back(); vec.push_back('2'); vec.push_back('5'); vec.push_back(c); vec.push_back(buf[0]); vec.push_back(buf[1]); encode_state = 0; } else { vec.push_back(c); encode_state = 0; } break; }}// get the next state with response to current state and current character// return false if come at '/0', '#', or '?'inline bool handle_path_normalize_state(int& state, char c, std::string& directory,std::list<std::string>& path) { switch (state) { case 0: // initial state if (c == '.') { state = 1; } else if (c == '/') { ///./,nothing to do } else if (c == '?' || c == '#' || c == 0) { // come at end return false; } else { directory.clear(); directory.push_back(c); state = 3; } break; case 1: // . if (c == '.') { state = 2; } else if (c == '?' || c == '#' || c == 0) { // /.[?#/0] state = 0; return false; } else if (c == '/') { ///./ state = 0; } else { directory.clear(); state = 3; } break; case 2: // .. if (c == '?' || c == '#' || c == 0) { // /./.[/?#/0], goto parent directory, and exit loop path.pop_back(); state = 0; return false; } else if (c == '/') { if (!path.empty()) { path.pop_back(); } state = 0; } else { directory.clear(); state = 3; } break; case 3: if (c == '?' || c == '#' || c == 0) { path.push_back(directory); state = 0; return false; } else if (c == '/') { directory.push_back('/'); path.push_back(directory); state = 0; } else { directory.push_back(c); } break; } return true;}// get the next state with response to the current state and current character// return false when come at 0,#inline bool handle_querystring_normalize_state(int& state, char c, std::string& key, std::string& value, std::map<std::string, std::string>& queries) {switch (state) {case 0: // initial stateif (c == '&') { // ignore} else if (c == 0 || c == '#') { // just exit return false;} else if (c == '=') { key.clear(); value.clear(); state = 2;} else {key.clear();key.push_back(c);state = 1;}break;case 1: // [^&/0#=]+if (c == '&') {queries[key] = "";state = 0;} else if (c == 0 || c == '#') { // set empty value, and exit queries[key] = "";state = 0;return false;} else if (c == '=') {value.clear();state = 2;} else {key.push_back(c);}break;case 2: // [^&/0#=]*=[^&/0#]*if (c == '&') { queries[key] = value; state = 0;} else if (c == 0 || c == '#') { queries[key] = value; state = 0; return false;} else {value.push_back(c);} break;}return true;}// normalize a file name : also called by robots.txt parser// return true if it is ok, false otherwise (cgi-bin)void fileNormalize(char* file, std::string* result) { int i = 0, j, state = 0, encode_state = 0; std::string directory; std::list<std::string> path; std::vector<char> vec; char c; int val = 0; for (; ;++i) { handle_url_encode_state(encode_state, file[i], val, vec); if (encode_state != 0) { continue; } if (vec.empty()) { vec.push_back(file[i]); } for (j = 0; j < vec.size(); ++j) { c = vec[j]; if (!handle_path_normalize_state(state, c, directory, path)) { goto lnFinishPath; } } vec.clear(); }lnFinishPath: // generates the file path result->push_back('/'); for (std::list<std::string>::iterator itr = path.begin(); itr != path.end();++itr) { (*result) += (*itr); } // if there exist query string if (c != '?') { return; } std::map<std::string, std::string> queries; std::string key, value; vec.clear(); for (++i; ;++i) { handle_url_encode_state(encode_state, file[i], val, vec); if (encode_state != 0) { continue; } if (vec.empty()) { vec.push_back(file[i]); } for (j = 0; j < vec.size(); ++j) { c = vec[j]; if (!handle_querystring_normalize_state(state, c, key, value, queries)) { goto lnFinishQueryString; } } vec.clear(); }lnFinishQueryString: int a = 0; // recover the query strings for (std::map<std::string, std::string>::iterator itr = queries.begin(); itr != queries.end(); ++itr) { if (itr == queries.begin()) { result->push_back('?'); result->append((itr->first).c_str()); result->push_back('='); result->append((itr->second).c_str()); } else { result->push_back('&'); result->append((itr->first).c_str()); result->push_back('='); result->append((itr->second).c_str()); } }} 

 

几个测试用例:

 

# 主机名称是否变成小写
slot {
  isgbk: false
  url: "http://www.ExAmple.com"
  base: "http://www.ExAmple.com"
  answer: "http://www.example.com/"
}

# 主机后面的点号是否去掉
slot {
  isgbk: false
  url: "http://sports.163.com../"
  base: "http://sports.163.com"
  answer: "http://sports.163.com/"
}

# 冗余的路径，非法路径消除,多余的/,.,..消除
slot {
  isgbk: false
  url: "http://sports.163.com/yc/../../../xj//index.html"
  base: "http://sports.163.com"
  answer: "http://sports.163.com/xj/index.html"
}

# 路径中的.和y被转义,是否再转回来
slot {
  isgbk: false
  url: "http://sports.163.com/%79c/%2e%2E/xj"
  base: "http://sports.163.com/"
  answer: "http://sports.163.com/xj"
}

# 不完整的url编码是否会被decode
slot {
  isgbk: false
  url: "http://www.baidu.com/s?wd=%%4%20abc"
  base: "http://www.baidu.com/"
  answer: "http://www.baidu.com/s?wd=%25%254+abc"
}

# 主机部分是否转成punycode
slot {
  isgbk: false
  url: "gsjs.htm"
  base: "http://北京风机厂.中国"
  answer: "http://xn--1lq90ihj568dev0d.xn--fiqs8s/gsjs.htm"
}

# 分段标志符是否去掉
slot {
  isgbk: false
  url: "http://www.example.com/bar.html#section1"
  base: "http://www.example.com"
  answer: "http://www.example.com/bar.html"
}

# querystring是否排序、消除了重复
slot {
  isgbk: false
  url: "http://www.example.com/display?lang=chn&article=fred&&lang=en"
  base: "http://www.example.com"
  answer: "http://www.example.com/display?article=fred&lang=en"
}

# 需要urlencode需转,不需要urlencode的要转回来,同时url编码部分必须用大写,空格统一用+
slot {
  isgbk: true
  url: "http://www.baidu.com/s?w%64=%4aack 你%20好"
  base: "http://www.baidU.com/"
  answer: "http://www.baidu.com/s?wd=Jack+%C4%E3+%BA%C3"
}