Style: Fix flake8 E261 at least two spaces before inline comment

[bookmarks_db.git] / parse_html / bkmk_ph_beautifulsoup4.py

diff --git a/parse_html/bkmk_ph_beautifulsoup4.py b/parse_html/bkmk_ph_beautifulsoup4.py
index aad3f8fb9092bdba2daec856a93f4309f9714795..b7f60ffd12033f787d6c4e6973324456948dce68 100644 (file)
--- a/parse_html/bkmk_ph_beautifulsoup4.py
+++ b/parse_html/bkmk_ph_beautifulsoup4.py
@@ -5,7 +5,7 @@ This file is a part of Bookmarks database and Internet robot.
 """
 
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2017 PhiloSoft Design"
+__copyright__ = "Copyright (C) 2017-2023 PhiloSoft Design"
 __license__ = "GNU GPL"
 
 __all__ = ['parse_html']
@@ -15,7 +15,8 @@ import re
 from bs4 import BeautifulSoup
 from .bkmk_ph_util import HTMLParser
 
-DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
+universal_charset = "utf-8"
+DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
 
 def _parse_html(html_text, charset):
     try:
@@ -35,7 +36,7 @@ def parse_html(html_text, charset=None, log=None):
 
     head = html.head
     if head is None:
-        head = html # Some sites put TITLE in HTML without HEAD
+        head = html  # Some sites put TITLE in HTML without HEAD
 
     title = head.title
     if (title is None) and (html is not head):
@@ -65,7 +66,7 @@ def parse_html(html_text, charset=None, log=None):
                 __charset = meta_content.lower().split('charset=')[1].split(';')[0]
             else:
                 __charset = False
-        except IndexError: # No charset in the META Content-Type
+        except IndexError:  # No charset in the META Content-Type
             meta_charset = False
         else:
             meta_charset = _charset = __charset
@@ -80,7 +81,11 @@ def parse_html(html_text, charset=None, log=None):
                 meta_charset = _charset = meta_content.lower()
 
     if title and (_charset or meta_charset):
-        title = title.encode(_charset or meta_charset)
+        try:
+            title = title.encode(_charset or meta_charset)
+        except LookupError:
+            title = title.encode(universal_charset)
+            _charset = universal_charset
 
     meta = head.find(_find_refresh, recursive=False)
     if meta: