From 53edc9fc59cc5785e3bef3b3bc6c7a3e825e8141 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 21 Feb 2025 17:34:04 +0300 Subject: [PATCH] Refactor(Robots): Split `base.py` into `util.py` --- Robots/base.py | 44 +--------------------------------------- Robots/util.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 43 deletions(-) create mode 100644 Robots/util.py diff --git a/Robots/base.py b/Robots/base.py index 3c51eac..5bf532f 100644 --- a/Robots/base.py +++ b/Robots/base.py @@ -12,8 +12,7 @@ __all__ = ['robot_base'] from base64 import b64encode -from urllib.parse import urlsplit, parse_qsl, quote, quote_plus, urlencode, \ - urljoin +from urllib.parse import urlsplit, urljoin import asyncio import sys import socket @@ -432,44 +431,3 @@ class robot_base(Robot): bookmark.last_tested = str(start) now = int(time.time()) bookmark.test_time = str(now - start) - - -def encode_url(url, encoding='latin1'): - split_results = urlsplit(url) - protocol, netloc, path, query, tag = split_results - user = split_results.username - password = split_results.password - host = split_results.hostname - port = split_results.port - - if query: - qlist = [] - for name, value in parse_qsl(query): - if not isinstance(name, bytes): - name = name.encode(encoding) - value = value.encode(encoding) - qlist.append((name, value)) - - url = protocol + "://" - if user: - url += quote(user.encode(encoding)) - if password: - url += ':' + quote(password.encode(encoding)) - url += '@' - if host: - if isinstance(host, bytes): - host = host.decode(encoding) - url += host.encode('idna').decode('ascii') - if port: - url += ':%d' % port - if path: - if protocol == "file": - url += quote(path) - else: - url += quote(path.encode(encoding)) - if query: - url += '?' + urlencode(qlist) - if tag: - url += '#' + quote_plus(tag.encode(encoding)) - - return url diff --git a/Robots/util.py b/Robots/util.py new file mode 100644 index 0000000..b978614 --- /dev/null +++ b/Robots/util.py @@ -0,0 +1,55 @@ +"""Utilities for robots + +This file is a part of Bookmarks database and Internet robot. + +""" + +__author__ = "Oleg Broytman " +__copyright__ = "Copyright (C) 2025 PhiloSoft Design" +__license__ = "GNU GPL" + +__all__ = ['encode_url'] + + +from urllib.parse import urlsplit, parse_qsl, quote, quote_plus, urlencode + + +def encode_url(url, encoding='latin1'): + split_results = urlsplit(url) + protocol, netloc, path, query, tag = split_results + user = split_results.username + password = split_results.password + host = split_results.hostname + port = split_results.port + + if query: + qlist = [] + for name, value in parse_qsl(query): + if not isinstance(name, bytes): + name = name.encode(encoding) + value = value.encode(encoding) + qlist.append((name, value)) + + url = protocol + "://" + if user: + url += quote(user.encode(encoding)) + if password: + url += ':' + quote(password.encode(encoding)) + url += '@' + if host: + if isinstance(host, bytes): + host = host.decode(encoding) + url += host.encode('idna').decode('ascii') + if port: + url += ':%d' % port + if path: + if protocol == "file": + url += quote(path) + else: + url += quote(path.encode(encoding)) + if query: + url += '?' + urlencode(qlist) + if tag: + url += '#' + quote_plus(tag.encode(encoding)) + + return url -- 2.39.5