#!/usr/bin/env python # coding:utf-8 # Based on GAppProxy 2.0.0 by Du XiaoGang # Based on WallProxy 0.4.0 by Hust Moon # Contributor: # Phus Lu # Hewig Xu # Ayanamist Yang # V.E.O # Max Lv # AlsoTang # Christopher Meng # Yonsm Guo # Parkman # Ming Bai # Bin Yu # lileixuan # Cong Ding # Zhang Youfu # Lu Wei # Harmony Meow # logostream # Rui Wang # Wang Wei Qiang # Felix Yan # Sui Feng # QXO # Geek An # Poly Rabbit # oxnz # Shusen Liu # Yad Smood # Chen Shuang # cnfuyu # cuixin # s2marine0 # Toshio Xiang __version__ = '3.1.5' import sys import os import glob sys.dont_write_bytecode = True sys.path += glob.glob('%s/*.egg' % os.path.dirname(os.path.abspath(__file__))) try: import gevent import gevent.socket import gevent.server import gevent.queue import gevent.monkey gevent.monkey.patch_all(subprocess=True) except ImportError: gevent = None except TypeError: gevent.monkey.patch_all() sys.stderr.write('\033[31m Warning: Please update gevent to the latest 1.0 version!\033[0m\n') import errno import binascii import time import struct import collections import zlib import functools import itertools import re import io import fnmatch import traceback import random import subprocess import base64 import string import hashlib import threading import thread import socket import ssl import select import Queue import SocketServer import ConfigParser import BaseHTTPServer import httplib import urllib2 import urlparse try: import dnslib except ImportError: dnslib = None try: import OpenSSL except ImportError: OpenSSL = None try: import pacparser except ImportError: pacparser = None HAS_PYPY = hasattr(sys, 'pypy_version_info') NetWorkIOError = (socket.error, ssl.SSLError, OSError) if not OpenSSL else (socket.error, ssl.SSLError, OpenSSL.SSL.Error, OSError) class Logging(type(sys)): CRITICAL = 50 FATAL = CRITICAL ERROR = 40 WARNING = 30 WARN = WARNING INFO = 20 DEBUG = 10 NOTSET = 0 def __init__(self, *args, **kwargs): self.level = self.__class__.INFO self.__set_error_color = lambda: None self.__set_warning_color = lambda: None self.__set_debug_color = lambda: None self.__reset_color = lambda: None if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): if os.name == 'nt': import ctypes SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute GetStdHandle = ctypes.windll.kernel32.GetStdHandle self.__set_error_color = lambda: SetConsoleTextAttribute(GetStdHandle(-11), 0x04) self.__set_warning_color = lambda: SetConsoleTextAttribute(GetStdHandle(-11), 0x06) self.__set_debug_color = lambda: SetConsoleTextAttribute(GetStdHandle(-11), 0x002) self.__reset_color = lambda: SetConsoleTextAttribute(GetStdHandle(-11), 0x07) elif os.name == 'posix': self.__set_error_color = lambda: sys.stderr.write('\033[31m') self.__set_warning_color = lambda: sys.stderr.write('\033[33m') self.__set_debug_color = lambda: sys.stderr.write('\033[32m') self.__reset_color = lambda: sys.stderr.write('\033[0m') @classmethod def getLogger(cls, *args, **kwargs): return cls(*args, **kwargs) def basicConfig(self, *args, **kwargs): self.level = int(kwargs.get('level', self.__class__.INFO)) if self.level > self.__class__.DEBUG: self.debug = self.dummy def log(self, level, fmt, *args, **kwargs): sys.stderr.write('%s - [%s] %s\n' % (level, time.ctime()[4:-5], fmt % args)) def dummy(self, *args, **kwargs): pass def debug(self, fmt, *args, **kwargs): self.__set_debug_color() self.log('DEBUG', fmt, *args, **kwargs) self.__reset_color() def info(self, fmt, *args, **kwargs): self.log('INFO', fmt, *args) def warning(self, fmt, *args, **kwargs): self.__set_warning_color() self.log('WARNING', fmt, *args, **kwargs) self.__reset_color() def warn(self, fmt, *args, **kwargs): self.warning(fmt, *args, **kwargs) def error(self, fmt, *args, **kwargs): self.__set_error_color() self.log('ERROR', fmt, *args, **kwargs) self.__reset_color() def exception(self, fmt, *args, **kwargs): self.error(fmt, *args, **kwargs) sys.stderr.write(traceback.format_exc() + '\n') def critical(self, fmt, *args, **kwargs): self.__set_error_color() self.log('CRITICAL', fmt, *args, **kwargs) self.__reset_color() logging = sys.modules['logging'] = Logging('logging') class LRUCache(object): """http://pypi.python.org/pypi/lru/""" def __init__(self, max_items=100): self.cache = {} self.key_order = [] self.max_items = max_items def __setitem__(self, key, value): self.cache[key] = value self._mark(key) def __getitem__(self, key): value = self.cache[key] self._mark(key) return value def _mark(self, key): if key in self.key_order: self.key_order.remove(key) self.key_order.insert(0, key) if len(self.key_order) > self.max_items: remove = self.key_order[self.max_items] del self.cache[remove] self.key_order.pop(self.max_items) def clear(self): self.cache = {} self.key_order = [] class CertUtil(object): """CertUtil module, based on mitmproxy""" ca_vendor = 'GoAgent' ca_keyfile = 'CA.crt' ca_certdir = 'certs' ca_lock = threading.Lock() @staticmethod def create_ca(): key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) ca = OpenSSL.crypto.X509() ca.set_serial_number(0) ca.set_version(2) subj = ca.get_subject() subj.countryName = 'CN' subj.stateOrProvinceName = 'Internet' subj.localityName = 'Cernet' subj.organizationName = CertUtil.ca_vendor subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor subj.commonName = '%s CA' % CertUtil.ca_vendor ca.gmtime_adj_notBefore(0) ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652) ca.set_issuer(ca.get_subject()) ca.set_pubkey(key) ca.add_extensions([ OpenSSL.crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE'), OpenSSL.crypto.X509Extension(b'nsCertType', True, b'sslCA'), OpenSSL.crypto.X509Extension(b'extendedKeyUsage', True, b'serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC'), OpenSSL.crypto.X509Extension(b'keyUsage', False, b'keyCertSign, cRLSign'), OpenSSL.crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=ca), ]) ca.sign(key, 'sha1') return key, ca @staticmethod def dump_ca(): key, ca = CertUtil.create_ca() with open(CertUtil.ca_keyfile, 'wb') as fp: fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca)) fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) @staticmethod def _get_cert(commonname, sans=()): with open(CertUtil.ca_keyfile, 'rb') as fp: content = fp.read() key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, content) ca = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content) pkey = OpenSSL.crypto.PKey() pkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) req = OpenSSL.crypto.X509Req() subj = req.get_subject() subj.countryName = 'CN' subj.stateOrProvinceName = 'Internet' subj.localityName = 'Cernet' subj.organizationalUnitName = '%s Branch' % CertUtil.ca_vendor if commonname[0] == '.': subj.commonName = '*' + commonname subj.organizationName = '*' + commonname sans = ['*'+commonname] + [x for x in sans if x != '*'+commonname] else: subj.commonName = commonname subj.organizationName = commonname sans = [commonname] + [x for x in sans if x != commonname] #req.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans)).encode()]) req.set_pubkey(pkey) req.sign(pkey, 'sha1') cert = OpenSSL.crypto.X509() cert.set_version(2) try: cert.set_serial_number(int(hashlib.md5(commonname.encode('utf-8')).hexdigest(), 16)) except OpenSSL.SSL.Error: cert.set_serial_number(int(time.time()*1000)) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(60 * 60 * 24 * 3652) cert.set_issuer(ca.get_subject()) cert.set_subject(req.get_subject()) cert.set_pubkey(req.get_pubkey()) if commonname[0] == '.': sans = ['*'+commonname] + [s for s in sans if s != '*'+commonname] else: sans = [commonname] + [s for s in sans if s != commonname] #cert.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans))]) cert.sign(key, 'sha1') certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt') with open(certfile, 'wb') as fp: fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, pkey)) return certfile @staticmethod def get_cert(commonname, sans=()): if commonname.count('.') >= 2 and len(commonname.split('.')[-2]) > 4: commonname = '.'+commonname.partition('.')[-1] certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt') if os.path.exists(certfile): return certfile elif OpenSSL is None: return CertUtil.ca_keyfile else: with CertUtil.ca_lock: if os.path.exists(certfile): return certfile return CertUtil._get_cert(commonname, sans) @staticmethod def import_ca(certfile): commonname = os.path.splitext(os.path.basename(certfile))[0] if OpenSSL: try: with open(certfile, 'rb') as fp: x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fp.read()) commonname = next(v.decode() for k, v in x509.get_subject().get_components() if k == b'O') except Exception as e: logging.error('load_certificate(certfile=%r) failed:%s', certfile, e) if sys.platform.startswith('win'): import ctypes with open(certfile, 'rb') as fp: certdata = fp.read() if certdata.startswith(b'-----'): begin = b'-----BEGIN CERTIFICATE-----' end = b'-----END CERTIFICATE-----' certdata = base64.b64decode(b''.join(certdata[certdata.find(begin)+len(begin):certdata.find(end)].strip().splitlines())) crypt32 = ctypes.WinDLL(b'crypt32.dll'.decode()) store_handle = crypt32.CertOpenStore(10, 0, 0, 0x4000 | 0x20000, b'ROOT'.decode()) if not store_handle: return -1 if crypt32.CertFindCertificateInStore(store_handle, 0x1, 0, 0x80007, CertUtil.ca_vendor.decode(), None): return 0 ret = crypt32.CertAddEncodedCertificateToStore(store_handle, 0x1, certdata, len(certdata), 4, None) crypt32.CertCloseStore(store_handle, 0) del crypt32 return 0 if ret else -1 elif sys.platform == 'darwin': return os.system(('security find-certificate -a -c "%s" | grep "%s" >/dev/null || security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "%s"' % (commonname, commonname, certfile.decode('utf-8'))).encode('utf-8')) elif sys.platform.startswith('linux'): import platform platform_distname = platform.dist()[0] if platform_distname == 'Ubuntu': pemfile = "/etc/ssl/certs/%s.pem" % commonname new_certfile = "/usr/local/share/ca-certificates/%s.crt" % commonname if not os.path.exists(pemfile): return os.system('cp "%s" "%s" && update-ca-certificates' % (certfile, new_certfile)) elif any(os.path.isfile('%s/certutil' % x) for x in os.environ['PATH'].split(os.pathsep)): return os.system('certutil -L -d sql:$HOME/.pki/nssdb | grep "%s" || certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (commonname, commonname, certfile)) else: logging.warning('please install *libnss3-tools* package to import GoAgent root ca') return 0 @staticmethod def check_ca(): #Check CA exists capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile) certdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_certdir) if not os.path.exists(capath): if not OpenSSL: logging.critical('CA.key is not exist and OpenSSL is disabled, ABORT!') sys.exit(-1) if os.path.exists(certdir): if os.path.isdir(certdir): any(os.remove(x) for x in glob.glob(certdir+'/*.crt')+glob.glob(certdir+'/.*.crt')) else: os.remove(certdir) os.mkdir(certdir) CertUtil.dump_ca() if glob.glob('%s/*.key' % CertUtil.ca_certdir): for filename in glob.glob('%s/*.key' % CertUtil.ca_certdir): try: os.remove(filename) os.remove(os.path.splitext(filename)[0]+'.crt') except EnvironmentError: pass #Check CA imported if CertUtil.import_ca(capath) != 0: logging.warning('install root certificate failed, Please run as administrator/root/sudo') #Check Certs Dir if not os.path.exists(certdir): os.makedirs(certdir) class SSLConnection(object): has_gevent = socket.socket is getattr(sys.modules.get('gevent.socket'), 'socket', None) def __init__(self, context, sock): self._context = context self._sock = sock self._connection = OpenSSL.SSL.Connection(context, sock) self._makefile_refs = 0 if self.has_gevent: self._wait_read = gevent.socket.wait_read self._wait_write = gevent.socket.wait_write self._wait_readwrite = gevent.socket.wait_readwrite else: self._wait_read = lambda fd,t: select.select([fd], [], [fd], t) self._wait_write = lambda fd,t: select.select([], [fd], [fd], t) self._wait_readwrite = lambda fd,t: select.select([fd], [fd], [fd], t) def __getattr__(self, attr): if attr not in ('_context', '_sock', '_connection', '_makefile_refs'): return getattr(self._connection, attr) def accept(self): sock, addr = self._sock.accept() client = OpenSSL.SSL.Connection(sock._context, sock) return client, addr def do_handshake(self): timeout = self._sock.gettimeout() while True: try: self._connection.do_handshake() break except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError, OpenSSL.SSL.WantWriteError): sys.exc_clear() self._wait_readwrite(self._sock.fileno(), timeout) def connect(self, *args, **kwargs): timeout = self._sock.gettimeout() while True: try: self._connection.connect(*args, **kwargs) break except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError): sys.exc_clear() self._wait_read(self._sock.fileno(), timeout) except OpenSSL.SSL.WantWriteError: sys.exc_clear() self._wait_write(self._sock.fileno(), timeout) def send(self, data, flags=0): timeout = self._sock.gettimeout() while True: try: self._connection.send(data, flags) break except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError): sys.exc_clear() self._wait_read(self._sock.fileno(), timeout) except OpenSSL.SSL.WantWriteError: sys.exc_clear() self._wait_write(self._sock.fileno(), timeout) except OpenSSL.SSL.SysCallError as e: if e[0] == -1 and not data: # errors when writing empty strings are expected and can be ignored return 0 raise def recv(self, bufsiz, flags=0): timeout = self._sock.gettimeout() pending = self._connection.pending() if pending: return self._connection.recv(min(pending, bufsiz)) while True: try: return self._connection.recv(bufsiz, flags) except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError): sys.exc_clear() self._wait_read(self._sock.fileno(), timeout) except OpenSSL.SSL.WantWriteError: sys.exc_clear() self._wait_write(self._sock.fileno(), timeout) except OpenSSL.SSL.ZeroReturnError: return '' def read(self, bufsiz, flags=0): return self.recv(bufsiz, flags) def write(self, buf, flags=0): return self.sendall(buf, flags) def close(self): if self._makefile_refs < 1: self._connection = None if self._sock: socket.socket.close(self._sock) else: self._makefile_refs -= 1 def makefile(self, mode='r', bufsize=-1): self._makefile_refs += 1 return socket._fileobject(self, mode, bufsize, close=True) class ProxyUtil(object): """ProxyUtil module, based on urllib2""" @staticmethod def parse_proxy(proxy): return urllib2._parse_proxy(proxy) @staticmethod def get_system_proxy(): proxies = urllib2.getproxies() return proxies.get('https') or proxies.get('http') or {} @staticmethod def get_listen_ip(): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.connect(('8.8.8.8', 53)) listen_ip = sock.getsockname()[0] sock.close() return listen_ip class PacUtil(object): """GoAgent Pac Util""" @staticmethod def update_pacfile(filename): listen_ip = ProxyUtil.get_listen_ip() if common.LISTEN_IP in ('', '::', '0.0.0.0') else common.LISTEN_IP autoproxy = '%s:%s' % (listen_ip, common.LISTEN_PORT) blackhole = '%s:%s' % (listen_ip, common.PAC_PORT) default = 'PROXY %s:%s' % (common.PROXY_HOST, common.PROXY_PORT) if common.PROXY_ENABLE else 'DIRECT' opener = urllib2.build_opener(urllib2.ProxyHandler({'http': autoproxy, 'https': autoproxy})) content = '' need_update = True with open(filename, 'rb') as fp: content = fp.read() try: placeholder = '// AUTO-GENERATED RULES, DO NOT MODIFY!' content = content[:content.index(placeholder)+len(placeholder)] content = re.sub(r'''blackhole\s*=\s*['"]PROXY [\.\w:]+['"]''', 'blackhole = \'PROXY %s\'' % blackhole, content) content = re.sub(r'''autoproxy\s*=\s*['"]PROXY [\.\w:]+['"]''', 'autoproxy = \'PROXY %s\'' % autoproxy, content) content = re.sub(r'''defaultproxy\s*=\s*['"](DIRECT|PROXY [\.\w:]+)['"]''', 'defaultproxy = \'%s\'' % default, content) content = re.sub(r'''host\s*==\s*['"][\.\w:]+['"]\s*\|\|\s*isPlainHostName''', 'host == \'%s\' || isPlainHostName' % listen_ip, content) if content.startswith('//'): line = '// Proxy Auto-Config file generated by autoproxy2pac, %s\r\n' % time.strftime('%Y-%m-%d %H:%M:%S') content = line + '\r\n'.join(content.splitlines()[1:]) except ValueError: need_update = False try: if common.PAC_ADBLOCK: logging.info('try download %r to update_pacfile(%r)', common.PAC_ADBLOCK, filename) adblock_content = opener.open(common.PAC_ADBLOCK).read() logging.info('%r downloaded, try convert it with adblock2pac', common.PAC_ADBLOCK) if 'gevent' in sys.modules and time.sleep is getattr(sys.modules['gevent'], 'sleep', None) and hasattr(gevent.get_hub(), 'threadpool'): jsrule = gevent.get_hub().threadpool.apply_e(Exception, PacUtil.adblock2pac, (adblock_content, 'FindProxyForURLByAdblock', blackhole, default)) else: jsrule = PacUtil.adblock2pac(adblock_content, 'FindProxyForURLByAdblock', blackhole, default) content += '\r\n' + jsrule + '\r\n' logging.info('%r downloaded and parsed', common.PAC_ADBLOCK) else: content += '\r\nfunction FindProxyForURLByAdblock(url, host) {return "DIRECT";}\r\n' except Exception as e: need_update = False logging.exception('update_pacfile failed: %r', e) try: logging.info('try download %r to update_pacfile(%r)', common.PAC_GFWLIST, filename) autoproxy_content = base64.b64decode(opener.open(common.PAC_GFWLIST).read()) logging.info('%r downloaded, try convert it with autoproxy2pac', common.PAC_GFWLIST) if 'gevent' in sys.modules and time.sleep is getattr(sys.modules['gevent'], 'sleep', None) and hasattr(gevent.get_hub(), 'threadpool'): jsrule = gevent.get_hub().threadpool.apply_e(Exception, PacUtil.autoproxy2pac, (autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default)) else: jsrule = PacUtil.autoproxy2pac(autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default) content += '\r\n' + jsrule + '\r\n' logging.info('%r downloaded and parsed', common.PAC_GFWLIST) except Exception as e: need_update = False logging.exception('update_pacfile failed: %r', e) if need_update: with open(filename, 'wb') as fp: fp.write(content) logging.info('%r successfully updated', filename) @staticmethod def autoproxy2pac(content, func_name='FindProxyForURLByAutoProxy', proxy='127.0.0.1:8087', default='DIRECT', indent=4): """Autoproxy to Pac, based on https://github.com/iamamac/autoproxy2pac""" jsLines = [] for line in content.splitlines()[1:]: if line and not line.startswith("!"): use_proxy = True if line.startswith("@@"): line = line[2:] use_proxy = False return_proxy = 'PROXY %s' % proxy if use_proxy else default if line.startswith('/') and line.endswith('/'): jsLine = 'if (/%s/i.test(url)) return "%s";' % (line[1:-1], return_proxy) elif line.startswith('||'): domain = line[2:].lstrip('.') if len(jsLines) > 0 and ('host.indexOf(".%s") >= 0' % domain in jsLines[-1] or 'host.indexOf("%s") >= 0' % domain in jsLines[-1]): jsLines.pop() jsLine = 'if (dnsDomainIs(host, ".%s") || host == "%s") return "%s";' % (domain, domain, return_proxy) elif line.startswith('|'): jsLine = 'if (url.indexOf("%s") == 0) return "%s";' % (line[1:], return_proxy) elif '*' in line: jsLine = 'if (shExpMatch(url, "*%s*")) return "%s";' % (line.strip('*'), return_proxy) elif '/' not in line: jsLine = 'if (host.indexOf("%s") >= 0) return "%s";' % (line, return_proxy) else: jsLine = 'if (url.indexOf("%s") >= 0) return "%s";' % (line, return_proxy) jsLine = ' ' * indent + jsLine if use_proxy: jsLines.append(jsLine) else: jsLines.insert(0, jsLine) function = 'function %s(url, host) {\r\n%s\r\n%sreturn "%s";\r\n}' % (func_name, '\n'.join(jsLines), ' '*indent, default) return function @staticmethod def urlfilter2pac(content, func_name='FindProxyForURLByUrlfilter', proxy='127.0.0.1:8086', default='DIRECT', indent=4): """urlfilter.ini to Pac, based on https://github.com/iamamac/autoproxy2pac""" jsLines = [] for line in content[content.index('[exclude]'):].splitlines()[1:]: if line and not line.startswith(';'): use_proxy = True if line.startswith("@@"): line = line[2:] use_proxy = False return_proxy = 'PROXY %s' % proxy if use_proxy else default if '*' in line: jsLine = 'if (shExpMatch(url, "%s")) return "%s";' % (line, return_proxy) else: jsLine = 'if (url == "%s") return "%s";' % (line, return_proxy) jsLine = ' ' * indent + jsLine if use_proxy: jsLines.append(jsLine) else: jsLines.insert(0, jsLine) function = 'function %s(url, host) {\r\n%s\r\n%sreturn "%s";\r\n}' % (func_name, '\n'.join(jsLines), ' '*indent, default) return function @staticmethod def adblock2pac(content, func_name='FindProxyForURLByAdblock', proxy='127.0.0.1:8086', default='DIRECT', indent=4): """adblock list to Pac, based on https://github.com/iamamac/autoproxy2pac""" jsLines = [] for line in content.splitlines()[1:]: if not line or line.startswith('!') or '##' in line or '#@#' in line: continue use_proxy = True use_start = False use_end = False use_domain = False use_postfix = [] if '$' in line: posfixs = line.split('$')[-1].split(',') if any('domain' in x for x in posfixs): continue if 'image' in posfixs: use_postfix += ['.jpg', '.gif'] elif 'script' in posfixs: use_postfix += ['.js'] else: continue line = line.split('$')[0] if line.startswith("@@"): line = line[2:] use_proxy = False if '||' == line[:2]: line = line[2:] if '/' not in line: use_domain = True else: if not line.startswith('http://'): line = 'http://' + line use_start = True elif '|' == line[0]: line = line[1:] if not line.startswith('http://'): line = 'http://' + line use_start = True if line[-1] in ('^', '|'): line = line[:-1] if not use_postfix: use_end = True return_proxy = 'PROXY %s' % proxy if use_proxy else default line = line.replace('^', '*').strip('*') if use_start and use_end: if '*' in line: jsLine = 'if (shExpMatch(url, "%s")) return "%s";' % (line, return_proxy) else: jsLine = 'if (url == "%s") return "%s";' % (line, return_proxy) elif use_start: if '*' in line: if use_postfix: jsCondition = ' || '.join('shExpMatch(url, "%s*%s")' % (line, x) for x in use_postfix) jsLine = 'if (%s) return "%s";' % (jsCondition, return_proxy) else: jsLine = 'if (shExpMatch(url, "%s*")) return "%s";' % (line, return_proxy) else: jsLine = 'if (url.indexOf("%s") == 0) return "%s";' % (line, return_proxy) elif use_domain and use_end: if '*' in line: jsLine = 'if (shExpMatch(host, "%s*")) return "%s";' % (line, return_proxy) else: jsLine = 'if (host == "%s") return "%s";' % (line, return_proxy) elif use_domain: if line.split('/')[0].count('.') <= 1: if use_postfix: jsCondition = ' || '.join('shExpMatch(url, "http://*.%s*%s")' % (line, x) for x in use_postfix) jsLine = 'if (%s) return "%s";' % (jsCondition, return_proxy) else: jsLine = 'if (shExpMatch(url, "http://*.%s*")) return "%s";' % (line, return_proxy) else: if '*' in line: if use_postfix: jsCondition = ' || '.join('shExpMatch(url, "http://%s*%s")' % (line, x) for x in use_postfix) jsLine = 'if (%s) return "%s";' % (jsCondition, return_proxy) else: jsLine = 'if (shExpMatch(url, "http://%s*")) return "%s";' % (line, return_proxy) else: if use_postfix: jsCondition = ' || '.join('shExpMatch(url, "http://%s*%s")' % (line, x) for x in use_postfix) jsLine = 'if (%s) return "%s";' % (jsCondition, return_proxy) else: jsLine = 'if (url.indexOf("http://%s") == 0) return "%s";' % (line, return_proxy) else: if use_postfix: jsCondition = ' || '.join('shExpMatch(url, "*%s*%s")' % (line, x) for x in use_postfix) jsLine = 'if (%s) return "%s";' % (jsCondition, return_proxy) else: jsLine = 'if (shExpMatch(url, "*%s*")) return "%s";' % (line, return_proxy) jsLine = ' ' * indent + jsLine.replace('**', '*') if use_proxy: jsLines.append(jsLine) else: jsLines.insert(0, jsLine) function = 'function %s(url, host) {\r\n%s\r\n%sreturn "%s";\r\n}' % (func_name, '\n'.join(jsLines), ' '*indent, default) return function def dns_remote_resolve(qname, dnsservers, blacklist, timeout): """ http://gfwrev.blogspot.com/2009/11/gfwdns.html http://zh.wikipedia.org/wiki/域名服务器缓存污染 http://support.microsoft.com/kb/241352 """ query = dnslib.DNSRecord(q=dnslib.DNSQuestion(qname)) query_data = query.pack() dns_v4_servers = [x for x in dnsservers if ':' not in x] dns_v6_servers = [x for x in dnsservers if ':' in x] sock_v4 = sock_v6 = None socks = [] if dns_v4_servers: sock_v4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) socks.append(sock_v4) if dns_v6_servers: sock_v6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) socks.append(sock_v6) timeout_at = time.time() + timeout try: for _ in xrange(2): try: for dnsserver in dns_v4_servers: sock_v4.sendto(query_data, (dnsserver, 53)) for dnsserver in dns_v6_servers: sock_v6.sendto(query_data, (dnsserver, 53)) while time.time() < timeout_at: ins, _, _ = select.select(socks, [], [], 0.1) for sock in ins: reply_data, _ = sock.recvfrom(512) reply = dnslib.DNSRecord.parse(reply_data) iplist = [str(x.rdata) for x in reply.rr if x.rtype == 1] if any(x in blacklist for x in iplist): logging.warning('query qname=%r reply bad iplist=%r', qname, iplist) else: logging.debug('query qname=%r reply iplist=%s', qname, iplist) return iplist except socket.error as e: logging.warning('handle dns query=%s socket: %r', query, e) finally: for sock in socks: sock.close() def spawn_later(seconds, target, *args, **kwargs): def wrap(*args, **kwargs): __import__('time').sleep(seconds) return target(*args, **kwargs) return __import__('thread').start_new_thread(wrap, args, kwargs) class HTTPUtil(object): """HTTP Request Class""" MessageClass = dict protocol_version = 'HTTP/1.1' skip_headers = frozenset(['Vary', 'Via', 'X-Forwarded-For', 'Proxy-Authorization', 'Proxy-Connection', 'Upgrade', 'X-Chrome-Variations', 'Connection', 'Cache-Control']) ssl_validate = False ssl_obfuscate = False ssl_ciphers = ':'.join(['ECDHE-ECDSA-AES256-SHA', 'ECDHE-RSA-AES256-SHA', 'DHE-RSA-CAMELLIA256-SHA', 'DHE-DSS-CAMELLIA256-SHA', 'DHE-RSA-AES256-SHA', 'DHE-DSS-AES256-SHA', 'ECDH-RSA-AES256-SHA', 'ECDH-ECDSA-AES256-SHA', 'CAMELLIA256-SHA', 'AES256-SHA', 'ECDHE-ECDSA-RC4-SHA', 'ECDHE-ECDSA-AES128-SHA', 'ECDHE-RSA-RC4-SHA', 'ECDHE-RSA-AES128-SHA', 'DHE-RSA-CAMELLIA128-SHA', 'DHE-DSS-CAMELLIA128-SHA', 'DHE-RSA-AES128-SHA', 'DHE-DSS-AES128-SHA', 'ECDH-RSA-RC4-SHA', 'ECDH-RSA-AES128-SHA', 'ECDH-ECDSA-RC4-SHA', 'ECDH-ECDSA-AES128-SHA', 'SEED-SHA', 'CAMELLIA128-SHA', 'RC4-SHA', 'RC4-MD5', 'AES128-SHA', 'ECDHE-ECDSA-DES-CBC3-SHA', 'ECDHE-RSA-DES-CBC3-SHA', 'EDH-RSA-DES-CBC3-SHA', 'EDH-DSS-DES-CBC3-SHA', 'ECDH-RSA-DES-CBC3-SHA', 'ECDH-ECDSA-DES-CBC3-SHA', 'DES-CBC3-SHA', 'TLS_EMPTY_RENEGOTIATION_INFO_SCSV']) def __init__(self, max_window=4, max_timeout=8, max_retry=4, proxy='', dns_servers=[], dns_blacklist=set()): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # http://www.openssl.org/docs/apps/ciphers.html # openssl s_server -accept 443 -key CA.crt -cert CA.crt # set_ciphers as Modern Browsers self.max_window = max_window self.max_retry = max_retry self.max_timeout = max_timeout self.tcp_connection_time = collections.defaultdict(float) self.tcp_connection_cache = collections.defaultdict(Queue.PriorityQueue) self.ssl_connection_time = collections.defaultdict(float) self.ssl_connection_cache = collections.defaultdict(Queue.PriorityQueue) self.dns = {} self.proxy = proxy self.openssl_context = None if self.proxy: self.dns_resolve = self.__dns_resolve_withproxy self.create_connection = self.__create_connection_withproxy self.create_ssl_connection = self.__create_ssl_connection_withproxy self.dns_servers = dns_servers self.dns_blacklist = dns_blacklist def set_openssl_option(self, validate=True, obfuscate=True): if self.openssl_context is None: self.openssl_context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) if validate: self.openssl_context.load_verify_locations(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cacert.pem')) self.openssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok) if obfuscate: ssl_ciphers = ':'.join(x for x in self.ssl_ciphers.split(':') if random.random() > 0.5) self.openssl_context.set_cipher_list(ssl_ciphers) def dns_resolve(self, host, dnsservers=[], ipv4_only=True): iplist = self.dns.get(host) if not iplist: if not dnsservers: iplist = list(set(socket.gethostbyname_ex(host)[-1]) - self.dns_blacklist) else: iplist = dns_remote_resolve(host, dnsservers, self.dns_blacklist, timeout=2) if not iplist: iplist = dns_remote_resolve(host, self.dns_servers, self.dns_blacklist, timeout=2) if ipv4_only: iplist = [ip for ip in iplist if re.match(r'\d+\.\d+\.\d+\.\d+', ip)] self.dns[host] = iplist = list(set(iplist)) return iplist def __dns_resolve_withproxy(self, host, dnsservers=[], ipv4_only=True): return [host] def create_connection(self, address, timeout=None, source_address=None, **kwargs): connection_cache_key = kwargs.get('cache_key') def _create_connection(ipaddr, timeout, queobj): sock = None try: # create a ipv4/ipv6 socket object sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable nagle algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout or self.max_timeout) # start connection time record start_time = time.time() # TCP connect sock.connect(ipaddr) # record TCP connection time self.tcp_connection_time[ipaddr] = time.time() - start_time # put ssl socket object to output queobj queobj.put(sock) except (socket.error, OSError) as e: # any socket.error, put Excpetions to output queobj. queobj.put(e) # reset a large and random timeout to the ipaddr self.tcp_connection_time[ipaddr] = self.max_timeout+random.random() # close tcp socket if sock: sock.close() def _close_connection(count, queobj): for i in range(count): sock = queobj.get() if sock and not isinstance(sock, Exception): if connection_cache_key and i == 0: self.tcp_connection_cache[connection_cache_key].put((time.time(), sock)) else: sock.close() try: while connection_cache_key: ctime, sock = self.tcp_connection_cache[connection_cache_key].get_nowait() if time.time() - ctime < 30: return sock except Queue.Empty: pass host, port = address result = None addresses = [(x, port) for x in self.dns_resolve(host)] if port == 443: get_connection_time = lambda addr: self.ssl_connection_time.__getitem__(addr) or self.tcp_connection_time.__getitem__(addr) else: get_connection_time = self.tcp_connection_time.__getitem__ for i in range(self.max_retry): window = min((self.max_window+1)//2 + min(i, 1), len(addresses)) addresses.sort(key=get_connection_time) addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window)) queobj = Queue.Queue() for addr in addrs: thread.start_new_thread(_create_connection, (addr, timeout, queobj)) for i in range(len(addrs)): result = queobj.get() if not isinstance(result, (socket.error, OSError)): thread.start_new_thread(_close_connection, (len(addrs)-i-1, queobj)) return result else: if i == 0: # only output first error logging.warning('create_connection to %s return %r, try again.', addrs, result) def create_ssl_connection(self, address, timeout=None, source_address=None, **kwargs): connection_cache_key = kwargs.get('cache_key') validate = kwargs.get('validate') def _create_ssl_connection(ipaddr, timeout, queobj): sock = None ssl_sock = None try: # create a ipv4/ipv6 socket object sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout or self.max_timeout) # pick up the certificate if not validate: ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False) else: ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=os.path.join(os.path.dirname(os.path.abspath(__file__)),'cacert.pem'), do_handshake_on_connect=False) ssl_sock.settimeout(timeout or self.max_timeout) # start connection time record start_time = time.time() # TCP connect ssl_sock.connect(ipaddr) connected_time = time.time() # SSL handshake ssl_sock.do_handshake() handshaked_time = time.time() # record TCP connection time self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time # record SSL connection time self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time ssl_sock.ssl_time = connected_time - start_time # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock # verify SSL certificate. if validate and address[0].endswith('.appspot.com'): cert = ssl_sock.getpeercert() orgname = next((v for ((k, v),) in cert['subject'] if k == 'organizationName')) if not orgname.lower().startswith('google '): raise ssl.SSLError("%r certificate organizationName(%r) not startswith 'Google'" % (address[0], orgname)) # put ssl socket object to output queobj queobj.put(ssl_sock) except (socket.error, ssl.SSLError, OSError) as e: # any socket.error, put Excpetions to output queobj. queobj.put(e) # reset a large and random timeout to the ipaddr self.ssl_connection_time[ipaddr] = self.max_timeout + random.random() # close ssl socket if ssl_sock: ssl_sock.close() # close tcp socket if sock: sock.close() def _create_openssl_connection(ipaddr, timeout, queobj): sock = None ssl_sock = None try: # create a ipv4/ipv6 socket object sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout or self.max_timeout) # pick up the certificate server_hostname = b'www.google.com' if address[0].endswith('.appspot.com') else None ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() if server_hostname: ssl_sock.set_tlsext_host_name(server_hostname) # start connection time record start_time = time.time() # TCP connect ssl_sock.connect(ipaddr) connected_time = time.time() # SSL handshake ssl_sock.do_handshake() handshaked_time = time.time() # record TCP connection time self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time # record SSL connection time self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock # verify SSL certificate. if validate and address[0].endswith('.appspot.com'): cert = ssl_sock.get_peer_certificate() commonname = next((v for k, v in cert.get_subject().get_components() if k == 'CN')) if '.google' not in commonname and not commonname.endswith('.appspot.com'): raise socket.error("Host name '%s' doesn't match certificate host '%s'" % (address[0], commonname)) # put ssl socket object to output queobj queobj.put(ssl_sock) except (socket.error, OpenSSL.SSL.Error, OSError) as e: # any socket.error, put Excpetions to output queobj. queobj.put(e) # reset a large and random timeout to the ipaddr self.ssl_connection_time[ipaddr] = self.max_timeout + random.random() # close ssl socket if ssl_sock: ssl_sock.close() # close tcp socket if sock: sock.close() def _close_ssl_connection(count, queobj, first_tcp_time, first_ssl_time): for i in range(count): sock = queobj.get() ssl_time_threshold = min(1, 1.5 * first_ssl_time) if sock and not isinstance(sock, Exception): if connection_cache_key and sock.ssl_time < ssl_time_threshold: self.ssl_connection_cache[connection_cache_key].put((time.time(), sock)) else: sock.close() try: while connection_cache_key: ctime, sock = self.ssl_connection_cache[connection_cache_key].get_nowait() if time.time() - ctime < 30: return sock except Queue.Empty: pass host, port = address result = None # create_connection = _create_ssl_connection if not validate else _create_openssl_connection create_connection = _create_ssl_connection addresses = [(x, port) for x in self.dns_resolve(host)] for i in range(self.max_retry): window = min((self.max_window+1)//2 + min(i, 1), len(addresses)) addresses.sort(key=self.ssl_connection_time.__getitem__) addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window)) queobj = Queue.Queue() for addr in addrs: thread.start_new_thread(create_connection, (addr, timeout, queobj)) for i in range(len(addrs)): result = queobj.get() if not isinstance(result, Exception): thread.start_new_thread(_close_ssl_connection, (len(addrs)-i-1, queobj, result.tcp_time, result.ssl_time)) return result else: if i == 0: # only output first error logging.warning('create_ssl_connection to %s return %r, try again.', addrs, result) def __create_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs): host, port = address logging.debug('__create_connection_withproxy connect (%r, %r)', host, port) _, proxyuser, proxypass, proxyaddress = ProxyUtil.parse_proxy(self.proxy) try: try: self.dns_resolve(host) except (socket.error, OSError): pass proxyhost, _, proxyport = proxyaddress.rpartition(':') sock = socket.create_connection((proxyhost, int(proxyport))) if host in self.dns: hostname = random.choice(self.dns[host]) elif host.endswith('.appspot.com'): hostname = 'www.google.com' else: hostname = host request_data = 'CONNECT %s:%s HTTP/1.1\r\n' % (hostname, port) if proxyuser and proxypass: request_data += 'Proxy-authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (proxyuser, proxypass)).encode()).decode().strip() request_data += '\r\n' sock.sendall(request_data) response = httplib.HTTPResponse(sock) response.begin() if response.status >= 400: logging.error('__create_connection_withproxy return http error code %s', response.status) sock = None return sock except Exception as e: logging.error('__create_connection_withproxy error %s', e) raise def __create_ssl_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs): host, port = address logging.debug('__create_ssl_connection_withproxy connect (%r, %r)', host, port) try: sock = self.__create_connection_withproxy(address, timeout, source_address) ssl_sock = ssl.wrap_socket(sock) ssl_sock.sock = sock return ssl_sock except Exception as e: logging.error('__create_ssl_connection_withproxy error %s', e) raise def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None): try: timecount = timeout while 1: timecount -= tick if timecount <= 0: break (ins, _, errors) = select.select([local, remote], [], [local, remote], tick) if errors: break if ins: for sock in ins: data = sock.recv(bufsize) if data: if sock is remote: local.sendall(data) timecount = maxpong or timeout else: remote.sendall(data) timecount = maxping or timeout else: return except NetWorkIOError as e: if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE): raise finally: if local: local.close() if remote: remote.close() def green_forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None, pongcallback=None, bitmask=None): def io_copy(dest, source): try: dest.settimeout(timeout) source.settimeout(timeout) while 1: data = source.recv(bufsize) if not data: break if bitmask: data = ''.join(chr(ord(x) ^ bitmask) for x in data) dest.sendall(data) except NetWorkIOError as e: if e.args[0] not in ('timed out', errno.ECONNABORTED, errno.ECONNRESET, errno.EBADF, errno.EPIPE, errno.ENOTCONN, errno.ETIMEDOUT): raise finally: if local: local.close() if remote: remote.close() thread.start_new_thread(io_copy, (remote.dup(), local.dup())) io_copy(local, remote) def _request(self, sock, method, path, protocol_version, headers, payload, bufsize=8192, crlf=None, return_sock=None): skip_headers = self.skip_headers need_crlf = bool(crlf) if need_crlf: fakehost = 'www.' + ''.join(random.choice(('bcdfghjklmnpqrstvwxyz','aeiou')[x&1]) for x in xrange(random.randint(5,20))) + random.choice(['.net', '.com', '.org']) request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n\r\n\r\r' % fakehost else: request_data = '' request_data += '%s %s %s\r\n' % (method, path, protocol_version) request_data += ''.join('%s: %s\r\n' % (k.title(), v) for k, v in headers.items() if k.title() not in skip_headers) if self.proxy: _, username, password, _ = ProxyUtil.parse_proxy(self.proxy) if username and password: request_data += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (username, password)).encode()).decode().strip() request_data += '\r\n' if isinstance(payload, bytes): sock.sendall(request_data.encode() + payload) elif hasattr(payload, 'read'): sock.sendall(request_data) while 1: data = payload.read(bufsize) if not data: break sock.sendall(data) else: raise TypeError('http_util.request(payload) must be a string or buffer, not %r' % type(payload)) if need_crlf: try: response = httplib.HTTPResponse(sock) response.begin() response.read() except Exception: logging.exception('crlf skip read') return None if return_sock: return sock response = httplib.HTTPResponse(sock, buffering=True) try: response.begin() except httplib.BadStatusLine: response = None return response def request(self, method, url, payload=None, headers={}, realhost='', bufsize=8192, crlf=None, validate=None, return_sock=None, connection_cache_key=None): scheme, netloc, path, _, query, _ = urlparse.urlparse(url) if netloc.rfind(':') <= netloc.rfind(']'): # no port number host = netloc port = 443 if scheme == 'https' else 80 else: host, _, port = netloc.rpartition(':') port = int(port) if query: path += '?' + query if 'Host' not in headers: headers['Host'] = host if payload and 'Content-Length' not in headers: headers['Content-Length'] = str(len(payload)) for i in range(self.max_retry): sock = None ssl_sock = None try: if scheme == 'https': ssl_sock = self.create_ssl_connection((realhost or host, port), self.max_timeout, validate=validate, cache_key=connection_cache_key) if ssl_sock: sock = ssl_sock.sock del ssl_sock.sock else: raise socket.error('timed out', 'create_ssl_connection(%r,%r)' % (realhost or host, port)) else: sock = self.create_connection((realhost or host, port), self.max_timeout, cache_key=connection_cache_key) if sock: if scheme == 'https': crlf = 0 return self._request(ssl_sock or sock, method, path, self.protocol_version, headers, payload, bufsize=bufsize, crlf=crlf, return_sock=return_sock) except Exception as e: logging.debug('request "%s %s" failed:%s', method, url, e) if ssl_sock: ssl_sock.close() if sock: sock.close() if i == self.max_retry - 1: raise else: continue class Common(object): """Global Config Object""" ENV_CONFIG_PREFIX = 'GOAGENT_' def __init__(self): """load config from proxy.ini""" ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P