| """Open an arbitrary URL. |
| |
| See the following document for more info on URLs: |
| "Names and Addresses, URIs, URLs, URNs, URCs", at |
| http://www.w3.org/pub/WWW/Addressing/Overview.html |
| |
| See also the HTTP spec (from which the error codes are derived): |
| "HTTP - Hypertext Transfer Protocol", at |
| http://www.w3.org/pub/WWW/Protocols/ |
| |
| Related standards and specs: |
| - RFC1808: the "relative URL" spec. (authoritative status) |
| - RFC1738 - the "URL standard". (authoritative status) |
| - RFC1630 - the "URI spec". (informational status) |
| |
| All code but that related to URL parsing has been removed (since it is not |
| compatible with Google App Engine)from this fork of the original file, |
| obtained from: |
| http://svn.python.org/view/*checkout*/python/tags/r252/Lib/urllib.py?content-type=text%2Fplain&rev=60915 |
| """ |
| |
| import string |
| import sys |
| from urlparse import urljoin as basejoin |
| |
| __all__ = ["quote", "quote_plus", "unquote", "unquote_plus", |
| "urlencode", "splittag", |
| "basejoin", "unwrap", |
| "splittype", "splithost", "splituser", "splitpasswd", "splitport", |
| "splitnport", "splitquery", "splitattr", "splitvalue", |
| "splitgophertype",] |
| |
| __version__ = '1.17' # XXX This version is not always updated :-( |
| |
| |
| # Utilities to parse URLs (most of these return None for missing parts): |
| # unwrap('<URL:type://host/path>') --> 'type://host/path' |
| # splittype('type:opaquestring') --> 'type', 'opaquestring' |
| # splithost('//host[:port]/path') --> 'host[:port]', '/path' |
| # splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]' |
| # splitpasswd('user:passwd') -> 'user', 'passwd' |
| # splitport('host:port') --> 'host', 'port' |
| # splitquery('/path?query') --> '/path', 'query' |
| # splittag('/path#tag') --> '/path', 'tag' |
| # splitattr('/path;attr1=value1;attr2=value2;...') -> |
| # '/path', ['attr1=value1', 'attr2=value2', ...] |
| # splitvalue('attr=value') --> 'attr', 'value' |
| # splitgophertype('/Xselector') --> 'X', 'selector' |
| # unquote('abc%20def') -> 'abc def' |
| # quote('abc def') -> 'abc%20def') |
| |
| try: |
| unicode |
| except NameError: |
| def _is_unicode(x): |
| return 0 |
| else: |
| def _is_unicode(x): |
| return isinstance(x, unicode) |
| |
| def toBytes(url): |
| """toBytes(u"URL") --> 'URL'.""" |
| # Most URL schemes require ASCII. If that changes, the conversion |
| # can be relaxed |
| if _is_unicode(url): |
| try: |
| url = url.encode("ASCII") |
| except UnicodeError: |
| raise UnicodeError("URL " + repr(url) + |
| " contains non-ASCII characters") |
| return url |
| |
| def unwrap(url): |
| """unwrap('<URL:type://host/path>') --> 'type://host/path'.""" |
| url = url.strip() |
| if url[:1] == '<' and url[-1:] == '>': |
| url = url[1:-1].strip() |
| if url[:4] == 'URL:': url = url[4:].strip() |
| return url |
| |
| _typeprog = None |
| def splittype(url): |
| """splittype('type:opaquestring') --> 'type', 'opaquestring'.""" |
| global _typeprog |
| if _typeprog is None: |
| import re |
| _typeprog = re.compile('^([^/:]+):') |
| |
| match = _typeprog.match(url) |
| if match: |
| scheme = match.group(1) |
| return scheme.lower(), url[len(scheme) + 1:] |
| return None, url |
| |
| _hostprog = None |
| def splithost(url): |
| """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" |
| global _hostprog |
| if _hostprog is None: |
| import re |
| _hostprog = re.compile('^//([^/?]*)(.*)$') |
| |
| match = _hostprog.match(url) |
| if match: return match.group(1, 2) |
| return None, url |
| |
| _userprog = None |
| def splituser(host): |
| """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" |
| global _userprog |
| if _userprog is None: |
| import re |
| _userprog = re.compile('^(.*)@(.*)$') |
| |
| match = _userprog.match(host) |
| if match: return map(unquote, match.group(1, 2)) |
| return None, host |
| |
| _passwdprog = None |
| def splitpasswd(user): |
| """splitpasswd('user:passwd') -> 'user', 'passwd'.""" |
| global _passwdprog |
| if _passwdprog is None: |
| import re |
| _passwdprog = re.compile('^([^:]*):(.*)$') |
| |
| match = _passwdprog.match(user) |
| if match: return match.group(1, 2) |
| return user, None |
| |
| # splittag('/path#tag') --> '/path', 'tag' |
| _portprog = None |
| def splitport(host): |
| """splitport('host:port') --> 'host', 'port'.""" |
| global _portprog |
| if _portprog is None: |
| import re |
| _portprog = re.compile('^(.*):([0-9]+)$') |
| |
| match = _portprog.match(host) |
| if match: return match.group(1, 2) |
| return host, None |
| |
| _nportprog = None |
| def splitnport(host, defport=-1): |
| """Split host and port, returning numeric port. |
| Return given default port if no ':' found; defaults to -1. |
| Return numerical port if a valid number are found after ':'. |
| Return None if ':' but not a valid number.""" |
| global _nportprog |
| if _nportprog is None: |
| import re |
| _nportprog = re.compile('^(.*):(.*)$') |
| |
| match = _nportprog.match(host) |
| if match: |
| host, port = match.group(1, 2) |
| try: |
| if not port: raise ValueError, "no digits" |
| nport = int(port) |
| except ValueError: |
| nport = None |
| return host, nport |
| return host, defport |
| |
| _queryprog = None |
| def splitquery(url): |
| """splitquery('/path?query') --> '/path', 'query'.""" |
| global _queryprog |
| if _queryprog is None: |
| import re |
| _queryprog = re.compile('^(.*)\?([^?]*)$') |
| |
| match = _queryprog.match(url) |
| if match: return match.group(1, 2) |
| return url, None |
| |
| _tagprog = None |
| def splittag(url): |
| """splittag('/path#tag') --> '/path', 'tag'.""" |
| global _tagprog |
| if _tagprog is None: |
| import re |
| _tagprog = re.compile('^(.*)#([^#]*)$') |
| |
| match = _tagprog.match(url) |
| if match: return match.group(1, 2) |
| return url, None |
| |
| def splitattr(url): |
| """splitattr('/path;attr1=value1;attr2=value2;...') -> |
| '/path', ['attr1=value1', 'attr2=value2', ...].""" |
| words = url.split(';') |
| return words[0], words[1:] |
| |
| _valueprog = None |
| def splitvalue(attr): |
| """splitvalue('attr=value') --> 'attr', 'value'.""" |
| global _valueprog |
| if _valueprog is None: |
| import re |
| _valueprog = re.compile('^([^=]*)=(.*)$') |
| |
| match = _valueprog.match(attr) |
| if match: return match.group(1, 2) |
| return attr, None |
| |
| def splitgophertype(selector): |
| """splitgophertype('/Xselector') --> 'X', 'selector'.""" |
| if selector[:1] == '/' and selector[1:2]: |
| return selector[1], selector[2:] |
| return None, selector |
| |
| _hextochr = dict(('%02x' % i, chr(i)) for i in range(256)) |
| _hextochr.update(('%02X' % i, chr(i)) for i in range(256)) |
| |
| def unquote(s): |
| """unquote('abc%20def') -> 'abc def'.""" |
| res = s.split('%') |
| for i in xrange(1, len(res)): |
| item = res[i] |
| try: |
| res[i] = _hextochr[item[:2]] + item[2:] |
| except KeyError: |
| res[i] = '%' + item |
| except UnicodeDecodeError: |
| res[i] = unichr(int(item[:2], 16)) + item[2:] |
| return "".join(res) |
| |
| def unquote_plus(s): |
| """unquote('%7e/abc+def') -> '~/abc def'""" |
| s = s.replace('+', ' ') |
| return unquote(s) |
| |
| always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| 'abcdefghijklmnopqrstuvwxyz' |
| '0123456789' '_.-') |
| _safemaps = {} |
| |
| def quote(s, safe = '/'): |
| """quote('abc def') -> 'abc%20def' |
| |
| Each part of a URL, e.g. the path info, the query, etc., has a |
| different set of reserved characters that must be quoted. |
| |
| RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists |
| the following reserved characters. |
| |
| reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | |
| "$" | "," |
| |
| Each of these characters is reserved in some component of a URL, |
| but not necessarily in all of them. |
| |
| By default, the quote function is intended for quoting the path |
| section of a URL. Thus, it will not encode '/'. This character |
| is reserved, but in typical usage the quote function is being |
| called on a path where the existing slash characters are used as |
| reserved characters. |
| """ |
| cachekey = (safe, always_safe) |
| try: |
| safe_map = _safemaps[cachekey] |
| except KeyError: |
| safe += always_safe |
| safe_map = {} |
| for i in range(256): |
| c = chr(i) |
| safe_map[c] = (c in safe) and c or ('%%%02X' % i) |
| _safemaps[cachekey] = safe_map |
| res = map(safe_map.__getitem__, s) |
| return ''.join(res) |
| |
| def quote_plus(s, safe = ''): |
| """Quote the query fragment of a URL; replacing ' ' with '+'""" |
| if ' ' in s: |
| s = quote(s, safe + ' ') |
| return s.replace(' ', '+') |
| return quote(s, safe) |
| |
| def urlencode(query,doseq=0): |
| """Encode a sequence of two-element tuples or dictionary into a URL query string. |
| |
| If any values in the query arg are sequences and doseq is true, each |
| sequence element is converted to a separate parameter. |
| |
| If the query arg is a sequence of two-element tuples, the order of the |
| parameters in the output will match the order of parameters in the |
| input. |
| """ |
| |
| if hasattr(query,"items"): |
| # mapping objects |
| query = query.items() |
| else: |
| # it's a bother at times that strings and string-like objects are |
| # sequences... |
| try: |
| # non-sequence items should not work with len() |
| # non-empty strings will fail this |
| if len(query) and not isinstance(query[0], tuple): |
| raise TypeError |
| # zero-length sequences of all types will get here and succeed, |
| # but that's a minor nit - since the original implementation |
| # allowed empty dicts that type of behavior probably should be |
| # preserved for consistency |
| except TypeError: |
| ty,va,tb = sys.exc_info() |
| raise TypeError, "not a valid non-string sequence or mapping object", tb |
| |
| l = [] |
| if not doseq: |
| # preserve old behavior |
| for k, v in query: |
| k = quote_plus(str(k)) |
| v = quote_plus(str(v)) |
| l.append(k + '=' + v) |
| else: |
| for k, v in query: |
| k = quote_plus(str(k)) |
| if isinstance(v, str): |
| v = quote_plus(v) |
| l.append(k + '=' + v) |
| elif _is_unicode(v): |
| # is there a reasonable way to convert to ASCII? |
| # encode generates a string, but "replace" or "ignore" |
| # lose information and "strict" can raise UnicodeError |
| v = quote_plus(v.encode("ASCII","replace")) |
| l.append(k + '=' + v) |
| else: |
| try: |
| # is this a sufficient test for sequence-ness? |
| x = len(v) |
| except TypeError: |
| # not a sequence |
| v = quote_plus(str(v)) |
| l.append(k + '=' + v) |
| else: |
| # loop over the sequence |
| for elt in v: |
| l.append(k + '=' + quote_plus(str(elt))) |
| return '&'.join(l) |
