X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=parse_html%2Fbkmk_ph_beautifulsoup4.py;h=182a123b402173d6a29c5e12d8971646541ba07f;hb=b9c8d112b8d8d0f7c726ee7dd07a89b6569c90a1;hp=10e06a97cd4fc4e25689cd3a92040878a5f2775a;hpb=be7cec61fa405f38dea2edde623174ca47ca7dc3;p=bookmarks_db.git diff --git a/parse_html/bkmk_ph_beautifulsoup4.py b/parse_html/bkmk_ph_beautifulsoup4.py index 10e06a9..182a123 100644 --- a/parse_html/bkmk_ph_beautifulsoup4.py +++ b/parse_html/bkmk_ph_beautifulsoup4.py @@ -5,7 +5,7 @@ This file is a part of Bookmarks database and Internet robot. """ __author__ = "Oleg Broytman " -__copyright__ = "Copyright (C) 2017 PhiloSoft Design" +__copyright__ = "Copyright (C) 2017-2023 PhiloSoft Design" __license__ = "GNU GPL" __all__ = ['parse_html'] @@ -16,7 +16,8 @@ from bs4 import BeautifulSoup from .bkmk_ph_util import HTMLParser universal_charset = "utf-8" -DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic +DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic + def _parse_html(html_text, charset): try: @@ -24,6 +25,7 @@ def _parse_html(html_text, charset): except TypeError: return None + def parse_html(html_text, charset=None, log=None): root = _parse_html(html_text, charset) if root is None: @@ -36,7 +38,7 @@ def parse_html(html_text, charset=None, log=None): head = html.head if head is None: - head = html # Some sites put TITLE in HTML without HEAD + head = html # Some sites put TITLE in HTML without HEAD title = head.title if (title is None) and (html is not head): @@ -66,7 +68,7 @@ def parse_html(html_text, charset=None, log=None): __charset = meta_content.lower().split('charset=')[1].split(';')[0] else: __charset = False - except IndexError: # No charset in the META Content-Type + except IndexError: # No charset in the META Content-Type meta_charset = False else: meta_charset = _charset = __charset @@ -103,17 +105,21 @@ def parse_html(html_text, charset=None, log=None): return None return HTMLParser(_charset, meta_charset, title, refresh, icon) + def _find_contenttype(Tag): return (Tag.name == "meta") and \ (Tag.get_attribute_list("http-equiv", '')[0].lower() == "content-type") + def _find_charset(Tag): return (Tag.name == "meta") and Tag.get("charset", '') + def _find_refresh(Tag): return (Tag.name == "meta") and \ (Tag.get_attribute_list("http-equiv", '')[0].lower() == "refresh") + def _find_icon(Tag): return (Tag.name == "link") and \ (Tag.get_attribute_list("rel", '')[0].lower() in ('icon', 'shortcut icon'))