#! /usr/bin/env python3
-try:
- PY2 = False
- from urllib.parse import parse_qsl, urlencode, \
- quote, quote_plus, unquote, unquote_plus, \
- splittype, splithost, splituser, splitpasswd, \
- splitport, splittag, splitquery
- from urllib.request import urlretrieve
- import urllib.request
-except ImportError:
- PY2 = True
- from cgi import parse_qsl
- from urllib import urlencode, quote, quote_plus, unquote, unquote_plus, \
- splittype, splithost, splituser, splitpasswd, \
- splitport, splittag, splitquery, urlretrieve
-
+from urllib.parse import urlsplit, parse_qsl, quote, quote_plus, urlencode
+from urllib.request import urlretrieve
import os
+import socket
import sys
-import urllib
-from m_lib.defenc import default_encoding
+import urllib.request
url = sys.argv[1]
-if PY2:
- _urlopener = urllib._urlopener = urllib.FancyURLopener()
-else:
- _urlopener = urllib.request._opener = urllib.request.FancyURLopener()
-
-protocol, request = splittype(url)
-user, password, port = None, None, None
-host, path = splithost(request)
-if host:
- user, host = splituser(host)
- if user:
- user, password = splitpasswd(user)
- host, port = splitport(host)
- if port: port = int(port)
-path, tag = splittag(path)
-path, query = splitquery(path)
-path = unquote(path)
-if tag: tag = unquote_plus(tag)
-if query:
- qlist = []
- for name, value in parse_qsl(query):
- qlist.append((name, value))
+split_results = urlsplit(url)
+protocol, netloc, path, query, tag = split_results
+user = split_results.username
+password = split_results.password
+host = split_results.hostname
+port = split_results.port
+qlist = parse_qsl(query)
url = protocol + "://"
if user:
url += ':' + quote(password)
url += '@'
if host:
- if PY2:
- host = host.decode(default_encoding)
- host = host.encode('idna')
- if not PY2:
- host = host.decode('ascii')
+ host = host.encode('idna').decode('ascii')
url += host
if port:
url += ':%d' % port
# without "compatible" user agent; I don't know if such sites are still around,
# but this header doesn't cause any harm so I'd better continue to use it.
# UPDATE: I saw a number of sites that forbid "Mozilla compatible"
-if PY2:
- urllib_version = urllib.__version__
-else:
- urllib_version = urllib.request.__version__
+urllib_version = urllib.request.__version__
server_version = "Python-urllib/%s" % urllib_version
+
+class MyURLopener(urllib.request.URLopener):
+ def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ return urllib.request.URLopener.open(self, fullurl, data)
+
+
+_urlopener = urllib.request._opener = MyURLopener()
_urlopener.addheaders[0] = ('User-agent', server_version)
_urlopener.addheaders.append(('Accept-Charset', "koi8-r;q=1.0"))