抓proxyhttp.net代理脚本

#!/usr/bin/env python  # -*- coding: utf-8 -*-    '''''     file: proxys.py     author: darkbull     date: 2011-08-01     desc:         从http://proxyhttp.net上抓取代码服务器地址 '''    import urllib2  import re  import time    _LASTEST_PROXY = '' # 最近一次获取的ip    def _get_html(url):      try:          headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",                "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.3",                # "Accept-Encoding": "gzip,deflate,sdch",              'Accept-Language': 'en-US,en;q=0.8',              'Cache-Control': 'max-age=0',              'Connection': 'keep-alive',              'Host': 'proxyhttp.net',              'User-Agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30',              'Referer': '',}          req = urllib2.Request(url, headers = headers)          conn = urllib2.urlopen(req, timeout = 10)          html = conn.read()          conn.close()          return html      except:          return ''          def _parse_html(html):      # 网页在计算端口值时做了手脚，例如端口80，在网站上表示为：52,44, 其实52 = ord('8') - 4, 0 = ord('0') - 4，这里的seed就是4      try:          seed = 0          pattern = r'String\.fromCharCode\((.+)\+parseInt\(a\[i\]\)\)'          m = re.search(pattern, html)          if m:              seed = int(m.groups(0)[0])          else:              raise Exception(u'known flag')            beg_tag = '<table class="proxytbl" cellSpacing="1">'          beg = html.index(beg_tag)  + len(beg_tag)          html = html[beg:]          end = html.index('</table>')          html = html[:end]                    pattern = r'<td class="t_ip">(.+?)</td>\s*<td class="t_port">(.+?)</td>\s*<td class="t_country"><img[^>]*/>(\w+)</td>[\s\S]+?<td class="t_https">(.*?)</td>'          _p = lambda port, seed: ''.join([chr(int(i) + seed) for i in port.split(',')])          return ['%s:%s,%s,%s' % (ip, _p(port, seed), controy, 'http' if https == '-' else 'https') for ip, port, controy, https in re.findall(pattern, html)]      except:          return []          def get_all_proxys():      '''''获取所有的代码服务器列表     Note: http://proxyhttp.net上只保存最近8页数据     '''      global _LASTEST_PROXY      pat = 'http://proxyhttp.net/free-list/anonymous-server-hide-ip-address/%d#proxylist'      ret = [ ]      for i in range(1, 3):          url = pat % i          html = _get_html(url)          if html:              ret.extend(_parse_html(html))          else:              if __debug__:                  print 'no html at:', url      if ret:          _LASTEST_PROXY = ret[0]      return ret          def get_lastest_proxys():      '''''获取最新添加的代理服务器     '''      global _LASTEST_PROXY      if not _LASTEST_PROXY:          return get_all_proxys()      else:          pat = 'http://proxyhttp.net/free-list/anonymous-server-hide-ip-address/%d#proxylist'          ret = [ ]          for i in range(1, 3):              url = pat % i              html = _get_html(url)              if html:                  t = _parse_html(html)                  if _LASTEST_PROXY in t:                      idx = t.index(_LASTEST_PROXY)                      if idx == 0:                          return ret                      t = t[:idx]                      _LASTEST_PROXY = t[0]                      ret.extend(t)                      return ret                  else:                      ret.extend(t)              else:                  if __debug__:                      print 'no html at:', url          return ret          if __name__ == '__main__':      import time      while True:          ret = get_lastest_proxys()          for i in ret:              print i                        print len(ret)          time.sleep(10)