From 1977729e7b34173edef8d4573532a8f753e79698 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Mon, 9 Sep 2024 23:10:56 +0300 Subject: [PATCH] Refactor(Robots): Move `encode_url` to `base.py` --- Robots/base.py | 44 +++++++++++++++++++++++++++++++++++++++++++- Robots/bkmk_rcurl.py | 44 +------------------------------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Robots/base.py b/Robots/base.py index 3aaac77..fa7178a 100644 --- a/Robots/base.py +++ b/Robots/base.py @@ -12,7 +12,8 @@ __all__ = ['robot_base'] from base64 import b64encode -from urllib.parse import urlsplit, urljoin +from urllib.parse import urlsplit, parse_qsl, quote, quote_plus, urlencode, \ + urljoin import asyncio import sys import socket @@ -429,3 +430,44 @@ class robot_base(Robot): bookmark.last_tested = str(start) now = int(time.time()) bookmark.test_time = str(now - start) + + +def encode_url(url, encoding='latin1'): + split_results = urlsplit(url) + protocol, netloc, path, query, tag = split_results + user = split_results.username + password = split_results.password + host = split_results.hostname + port = split_results.port + + if query: + qlist = [] + for name, value in parse_qsl(query): + if not isinstance(name, bytes): + name = name.encode(encoding) + value = value.encode(encoding) + qlist.append((name, value)) + + url = protocol + "://" + if user: + url += quote(user.encode(encoding)) + if password: + url += ':' + quote(password.encode(encoding)) + url += '@' + if host: + if isinstance(host, bytes): + host = host.decode(encoding) + url += host.encode('idna').decode('ascii') + if port: + url += ':%d' % port + if path: + if protocol == "file": + url += quote(path) + else: + url += quote(path.encode(encoding)) + if query: + url += '?' + urlencode(qlist) + if tag: + url += '#' + quote_plus(tag.encode(encoding)) + + return url diff --git a/Robots/bkmk_rcurl.py b/Robots/bkmk_rcurl.py index 36c715b..553dc7f 100644 --- a/Robots/bkmk_rcurl.py +++ b/Robots/bkmk_rcurl.py @@ -14,14 +14,13 @@ __all__ = ['robot_curl'] from queue import Queue from time import sleep -from urllib.parse import urlsplit, parse_qsl, quote, quote_plus, urlencode import concurrent.futures import threading import certifi import pycurl -from Robots.base import robot_base +from Robots.base import robot_base, encode_url from Robots.concurrent_futures import concurrent_futures @@ -172,44 +171,3 @@ class CurlWrapper: def body_callback(self, data): self.body += data - - -def encode_url(url, encoding='latin1'): - split_results = urlsplit(url) - protocol, netloc, path, query, tag = split_results - user = split_results.username - password = split_results.password - host = split_results.hostname - port = split_results.port - - if query: - qlist = [] - for name, value in parse_qsl(query): - if not isinstance(name, bytes): - name = name.encode(encoding) - value = value.encode(encoding) - qlist.append((name, value)) - - url = protocol + "://" - if user: - url += quote(user.encode(encoding)) - if password: - url += ':' + quote(password.encode(encoding)) - url += '@' - if host: - if isinstance(host, bytes): - host = host.decode(encoding) - url += host.encode('idna').decode('ascii') - if port: - url += ':%d' % port - if path: - if protocol == "file": - url += quote(path) - else: - url += quote(path.encode(encoding)) - if query: - url += '?' + urlencode(qlist) - if tag: - url += '#' + quote_plus(tag.encode(encoding)) - - return url -- 2.39.5