From f0cc1cb95379c0d8a18f2511bf5ccbb21f946f5f Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 12 Jun 2014 17:40:00 +0400 Subject: [PATCH] Adapt JSON storage to recent Mozilla export format Adapt JSON storage to recent export format. Tested with Firefox 29 and 30. Remove root folder's attributes. Remove 'parent' key. Process 'guid', and 'iconuri' keys. Store timestamps as floats. Special handling for 'places:' "bookmarks". --- Storage/bkmk_stjson.py | 64 ++++++++++++++++++------------------------ doc/ChangeLog | 2 ++ 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/Storage/bkmk_stjson.py b/Storage/bkmk_stjson.py index 53eae9b..a08f619 100644 --- a/Storage/bkmk_stjson.py +++ b/Storage/bkmk_stjson.py @@ -1,10 +1,11 @@ """Bookmarks storage manager - json This file is a part of Bookmarks database and Internet robot. + """ __author__ = "Oleg Broytman " -__copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design" +__copyright__ = "Copyright (C) 2010-2014 PhiloSoft Design" __license__ = "GNU GPL" __all__ = ['storage_json'] @@ -25,12 +26,6 @@ class storage_json(Walker): self.dict = dict = {} dict["children"] = children = [] self.folder_stack = [children] - dict["dateAdded"] = convert_date_to_json(f.add_date) - dict["id"] = f.id - dict["lastModified"] = convert_date_to_json(f.last_modified) - dict["root"] = "placesRoot" - dict["title"] = "" - dict["type"] = "text/x-moz-place-container" def start_folder(self, f, level): dict = {} @@ -38,17 +33,20 @@ class storage_json(Walker): if comment: dict["annos"] = make_annos(comment) dict["children"] = children = [] dict["dateAdded"] = convert_date_to_json(f.add_date) + guid = getattr(f, 'guid') + if guid: dict["guid"] = guid dict["id"] = f.id index = getattr(f, 'index') - if index: dict["index"] = index + if index is not None: dict["index"] = index dict["lastModified"] = convert_date_to_json(f.last_modified) - parent_idx = getattr(f, 'parent_idx') - if parent_idx: dict["parent"] = parent_idx root = getattr(f, 'root') if root: dict["root"] = root dict["title"] = f.name.decode('utf-8') dict["type"] = "text/x-moz-place-container" - self.folder_stack[-1].append(dict) + if root: + self.dict["children"].append(dict) + else: + self.folder_stack[-1].append(dict) self.folder_stack.append(children) def end_folder(self, f, level): @@ -61,30 +59,19 @@ class storage_json(Walker): charset = getattr(b, 'charset') if charset: dict["charset"] = charset dict["dateAdded"] = convert_date_to_json(b.add_date) + guid = getattr(b, 'guid') + if guid: dict["guid"] = guid dict["id"] = b.id + iconuri = getattr(b, 'icon_href') + if iconuri: dict["iconuri"] = iconuri index = getattr(b, 'index') - if index: dict["index"] = index + if index is not None: dict["index"] = index keyword = getattr(b, 'keyword') if keyword: dict["keyword"] = keyword dict["lastModified"] = convert_date_to_json(b.last_modified) - dict["parent"] = b.parent_idx dict["title"] = b.name.decode('utf-8') dict["type"] = "text/x-moz-place" dict["uri"] = uri = b.href - if uri.startswith('place:'): - if uri.startswith('place:folder'): - value = 'Recently Bookmarked' - elif uri.startswith('place:sort=8') or \ - uri.startswith('place:redirectsMode'): - value = 'Most Visited' - elif uri.startswith('place:sort=14') or \ - uri.startswith('place:type=6'): - value = 'Recent Tags' - else: - raise ValueError('UNKNOWN place URI: %s' % uri) - dict["annos"] = make_annos(value, name='Places/SmartBookmark') - del dict["dateAdded"] - del dict["lastModified"] self.folder_stack[-1].append(dict) def ruler(self, r, level): @@ -93,9 +80,10 @@ class storage_json(Walker): if comment: dict["annos"] = make_annos(comment) dict["dateAdded"] = convert_date_to_json(r.add_date) dict["id"] = r.id + guid = getattr(r, 'guid') + if guid: dict["guid"] = guid dict["index"] = r.index dict["lastModified"] = convert_date_to_json(r.last_modified) - dict["parent"] = r.parent_idx dict["title"] = r.name.decode('utf-8') dict["type"] = "text/x-moz-place-separator" self.folder_stack[-1].append(dict) @@ -127,7 +115,9 @@ class storage_json(Walker): self.current_folder = root_folder if "type" not in bookmarks_dict: - bookmarks_dict = bookmarks_dict["children"][0] + bookmarks_dict["id"] = "0" + bookmarks_dict["title"] = "" + bookmarks_dict["type"] = "text/x-moz-place-container" self.load_folder(root_folder, bookmarks_dict) if self.folder_stack: raise RuntimeError('Excessive folder stack: %s' % self.folder_stack) @@ -139,8 +129,8 @@ class storage_json(Walker): raise ValueError("The object is not a Mozilla container") folder.id = fdict["id"] + folder.guid = fdict.get("guid") folder.index = fdict.get("index") - folder.parent_idx = fdict.get("parent") folder.root = fdict.get("root") folder.name = encode_title(fdict["title"]) @@ -150,6 +140,7 @@ class storage_json(Walker): add_date=convert_date_from_json(record.get("dateAdded")), comment=get_comment(record.get("annos")), last_modified=convert_date_from_json(record.get("lastModified"))) + folder.guid = record.get("guid") self.current_folder.append(folder) self.folder_stack.append(folder) self.current_folder = folder @@ -162,20 +153,21 @@ class storage_json(Walker): last_modified=convert_date_from_json(record.get("lastModified")), keyword=get_str(record, "keyword"), comment=get_comment(record.get("annos")), + icon_href=record.get("iconuri"), charset=get_str(record, "charset")) + bookmark.guid = record.get("guid") bookmark.id = record["id"] bookmark.index = record.get("index") - bookmark.parent_idx = record.get("parent") bookmark.name = encode_title(record["title"]) self.current_folder.append(bookmark) elif record["type"] == "text/x-moz-place-separator": ruler = Ruler() ruler.add_date = convert_date_from_json(record.get("dateAdded")) + ruler.guid = record.get("guid") ruler.id = record["id"] ruler.index = record["index"] ruler.last_modified = convert_date_from_json(record.get("lastModified")) - ruler.parent_idx = record.get("parent") ruler.name = encode_title(record["title"]) ruler.comment = get_comment(record.get("annos")) self.current_folder.append(ruler) @@ -192,14 +184,14 @@ class storage_json(Walker): def convert_date_to_json(date): if date: - date = int(date) * 10**6 + date = int(date * 10**6) return date def convert_date_from_json(date): if date: - date = int(date) + date = float(date) if date > 10**10: - date /= 10**6 + date /= 10.0**6 return date def encode_title(title): @@ -224,7 +216,5 @@ def make_annos(value, name="bookmarkProperties/description"): return [{ "expires": 4, "flags": 0, - "mimeType": None, "name": name, - "type": 3, "value": value.decode('utf-8')}] diff --git a/doc/ChangeLog b/doc/ChangeLog index d6e6a7e..62879a3 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -3,6 +3,8 @@ WHAT'S NEW in version 4.6.0 Split simple robot: separate network operations and URL handling/HTML parsing. + Adapt JSON storage to recent Mozilla export format. + WHAT'S NEW in version 4.5.7 -- 2.39.5