Remove robots based on Twisted

[bookmarks_db.git] / parse_html / bkmk_ph_beautifulsoup4.py

diff --git a/parse_html/bkmk_ph_beautifulsoup4.py b/parse_html/bkmk_ph_beautifulsoup4.py
index 1095ebce8d510c22df2c9768baed3c27777e1909..47c313091399880971577c5c56138100cf769824 100644 (file)
--- a/parse_html/bkmk_ph_beautifulsoup4.py
+++ b/parse_html/bkmk_ph_beautifulsoup4.py
@@ -5,16 +5,21 @@ This file is a part of Bookmarks database and Internet robot.
 """
 
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2017-2023 PhiloSoft Design"
+__copyright__ = "Copyright (C) 2017-2025 PhiloSoft Design"
 __license__ = "GNU GPL"
 
 __all__ = ['parse_html']
 
 
+import warnings
+
 from bs4 import BeautifulSoup
+from bs4 import XMLParsedAsHTMLWarning
 
 from .bkmk_ph_util import HTMLParser
-from compat import string_type
+
+warnings.filterwarnings('ignore', 'No parser was explicitly specified')
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
 
 universal_charset = "utf-8"
 DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
@@ -22,12 +27,17 @@ DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
 
 def _parse_html(html_text, charset):
     try:
-        return BeautifulSoup(html_text, from_encoding=charset)
+        if isinstance(html_text, bytes):
+            return BeautifulSoup(html_text, from_encoding=charset)
+        else:
+            return BeautifulSoup(html_text)
     except TypeError:
         return None
 
 
 def parse_html(html_text, charset=None, log=None):
+    if not html_text:
+        return None
     root = _parse_html(html_text, charset)
     if root is None:
         return None
@@ -56,9 +66,10 @@ def parse_html(html_text, charset=None, log=None):
         else:
             parts = []
             for part in title:
-                if not isinstance(part, string_type):
-                    part = part.decode()
-                parts.append(part.strip())
+                if part.strip:
+                    parts.append(part.strip())
+                else:
+                    parts.append(' ')  # Skip tags, they're usually `<br>`
             title = ''.join(parts)
 
     meta = head.find(_find_contenttype, recursive=False)
@@ -84,13 +95,6 @@ def parse_html(html_text, charset=None, log=None):
             if meta_content:
                 meta_charset = _charset = meta_content.lower()
 
-    #if title and (_charset or meta_charset):
-    #    try:
-    #        title = title.encode(_charset or meta_charset)
-    #    except LookupError:
-    #        title = title.encode(universal_charset)
-    #        _charset = universal_charset
-
     meta = head.find(_find_refresh, recursive=False)
     if meta:
         refresh = meta.get("content")
@@ -123,6 +127,7 @@ def _find_refresh(Tag):
 
 
 def _find_icon(Tag):
-    return (Tag.name == "link") and \
-       (Tag.get_attribute_list("rel", '')[0].lower()
-        in ('icon', 'shortcut icon'))
+    if Tag.name != "link":
+        return False
+    rel = ' '.join(Tag.get_attribute_list("rel", ''))
+    return rel in ('icon', 'shortcut icon')