X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fparse_html.py;h=d3870fa210b8c3bdeefdac3c45d05aa13ec6cc1a;hb=cbe95b9700da2b5b75fa07312253736d5447d318;hp=c5c9eb6831e0db333cf95cd25314dbc453a5e28a;hpb=4c8bc9dc1f441cf9256ce3a933f51213a0c3c3ff;p=bookmarks_db.git

diff --git a/Robots/parse_html.py b/Robots/parse_html.py
index c5c9eb6..d3870fa 100755
--- a/Robots/parse_html.py
+++ b/Robots/parse_html.py
@@ -30,7 +30,10 @@ def recode_entities(title, charset):
    output = []
    for part in entity_re.split(title):
       if entity_re.match(part):
-         part = unichr(int(part[2:-1])).encode(charset, "replace")
+         try:
+            part = unichr(int(part[2:-1])).encode(charset)
+         except UnicodeEncodeError:
+            pass # Leave the entity as is
       output.append(part)
 
    return ''.join(output)
@@ -84,7 +87,8 @@ def parse_html(filename, charset=None, log=None):
          if log: log("   unknown charset: `%s' or `%s'" % (parser.charset, current_charset))
 
    title = recode_entities(title, current_charset)
-   title = title.replace('\r', '').replace('\n', ' ').strip()
+   parts = [s.strip() for s in title.replace('\r', '').split('\n')]
+   title = ' '.join([s for s in parts if s])
    if log: log("   final title    : %s" % title)
    parser.title = title
    return parser