From e7b88f0546a9b954a74149f3bfae965a3edae547 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 1 Aug 2024 13:03:16 +0300 Subject: [PATCH] Feat(Robots): Update X-User-Agent header --- Robots/bkmk_rcurl.py | 3 +++ Robots/bkmk_robot_base.py | 7 +++++++ Robots/bkmk_rrequests.py | 4 ++++ Robots/bkmk_rurllib.py | 3 +++ Robots/bkmk_rurllib2.py | 3 +++ Robots/bkmk_rurllib_py3.py | 3 +++ 6 files changed, 23 insertions(+) diff --git a/Robots/bkmk_rcurl.py b/Robots/bkmk_rcurl.py index 6125856..96f208c 100644 --- a/Robots/bkmk_rcurl.py +++ b/Robots/bkmk_rcurl.py @@ -18,6 +18,9 @@ from Robots.bkmk_robot_base import robot_base, request_headers class robot_curl(robot_base): + def version_str(self): + return str(pycurl.version) + def get(self, bookmark, url, accept_charset=False, use_proxy=False): if accept_charset and bookmark.charset: headers = request_headers.copy() diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index 2d186ac..9b989e5 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -84,6 +84,13 @@ class robot_base(Robot): Robot.__init__(self, *args, **kw) socket.setdefaulttimeout(int(self.timeout)) + global _x_user_agent + _x_user_agent = '%s %s' % (_x_user_agent, self.version_str()) + request_headers['X-User-Agent'] = _x_user_agent + + def version_str(self): + return self.__class__.__name__ + def check_url(self, bookmark): try: self.start = int(time.time()) diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py index e27c411..638ed91 100644 --- a/Robots/bkmk_rrequests.py +++ b/Robots/bkmk_rrequests.py @@ -17,6 +17,7 @@ from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.ssl_ import create_urllib3_context import requests import requests_ftp +import urllib3 from Robots.bkmk_robot_base import robot_base, request_headers @@ -24,6 +25,9 @@ requests_ftp.monkeypatch_session() class robot_requests(robot_base): + def version_str(self): + return 'python-requests urllib3/%s' % urllib3.__version__ + def get(self, bookmark, url, accept_charset=False, use_proxy=False): if accept_charset and bookmark.charset: headers = request_headers.copy() diff --git a/Robots/bkmk_rurllib.py b/Robots/bkmk_rurllib.py index 19d9fc1..2731ad5 100644 --- a/Robots/bkmk_rurllib.py +++ b/Robots/bkmk_rurllib.py @@ -84,6 +84,9 @@ urllib.ftpwrapper = myftpwrapper class robot_urllib(robot_base): + def version_str(self): + return 'urllib' + def get(self, bookmark, url, accept_charset=False, use_proxy=False): if use_proxy: opener.proxies = {'http': self.proxy, 'https': self.proxy} diff --git a/Robots/bkmk_rurllib2.py b/Robots/bkmk_rurllib2.py index 5e60497..13d2c3b 100644 --- a/Robots/bkmk_rurllib2.py +++ b/Robots/bkmk_rurllib2.py @@ -39,6 +39,9 @@ urllib2.install_opener(opener) class robot_urllib2(robot_base): + def version_str(self): + return 'urllib2' + def get(self, bookmark, url, accept_charset=False, use_proxy=False): request = urllib2.Request(url) for h, v in request_headers.items(): diff --git a/Robots/bkmk_rurllib_py3.py b/Robots/bkmk_rurllib_py3.py index 3928768..7947046 100644 --- a/Robots/bkmk_rurllib_py3.py +++ b/Robots/bkmk_rurllib_py3.py @@ -85,6 +85,9 @@ urllib.request.ftpwrapper = myftpwrapper class robot_urllib_py3(robot_base): + def version_str(self): + return 'urllib' + def get(self, bookmark, url, accept_charset=False, use_proxy=False): if use_proxy: opener.proxies = {'http': self.proxy, 'https': self.proxy} -- 2.39.5