Adapt JSON storage to recent Mozilla export format

Adapt JSON storage to recent export format. Tested with Firefox 29 and 30.
Remove root folder's attributes. Remove 'parent' key.
Process 'guid', and 'iconuri' keys. Store timestamps as floats.
Special handling for 'places:' "bookmarks".

diff --git a/Storage/bkmk_stjson.py b/Storage/bkmk_stjson.py
index 53eae9b81c14b9a316ed6a46dc738cf81b5b7a5f..a08f61910a4415d76b8b64c51a0443070d4b3cbf 100644 (file)
--- a/Storage/bkmk_stjson.py
+++ b/Storage/bkmk_stjson.py
@@ -1,10 +1,11 @@
 """Bookmarks storage manager - json
 
 This file is a part of Bookmarks database and Internet robot.
+
 """
 
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design"
+__copyright__ = "Copyright (C) 2010-2014 PhiloSoft Design"
 __license__ = "GNU GPL"
 
 __all__ = ['storage_json']
@@ -25,12 +26,6 @@ class storage_json(Walker):
         self.dict = dict = {}
         dict["children"] = children = []
         self.folder_stack = [children]
-        dict["dateAdded"] = convert_date_to_json(f.add_date)
-        dict["id"] = f.id
-        dict["lastModified"] = convert_date_to_json(f.last_modified)
-        dict["root"] = "placesRoot"
-        dict["title"] = ""
-        dict["type"] = "text/x-moz-place-container"
 
     def start_folder(self, f, level):
         dict = {}
@@ -38,17 +33,20 @@ class storage_json(Walker):
         if comment: dict["annos"] = make_annos(comment)
         dict["children"] = children = []
         dict["dateAdded"] = convert_date_to_json(f.add_date)
+        guid = getattr(f, 'guid')
+        if guid: dict["guid"] = guid
         dict["id"] = f.id
         index = getattr(f, 'index')
-        if index: dict["index"] = index
+        if index is not None: dict["index"] = index
         dict["lastModified"] = convert_date_to_json(f.last_modified)
-        parent_idx = getattr(f, 'parent_idx')
-        if parent_idx: dict["parent"] = parent_idx
         root = getattr(f, 'root')
         if root: dict["root"] = root
         dict["title"] = f.name.decode('utf-8')
         dict["type"] = "text/x-moz-place-container"
-        self.folder_stack[-1].append(dict)
+        if root:
+            self.dict["children"].append(dict)
+        else:
+            self.folder_stack[-1].append(dict)
         self.folder_stack.append(children)
 
     def end_folder(self, f, level):
@@ -61,30 +59,19 @@ class storage_json(Walker):
         charset = getattr(b, 'charset')
         if charset: dict["charset"] = charset
         dict["dateAdded"] = convert_date_to_json(b.add_date)
+        guid = getattr(b, 'guid')
+        if guid: dict["guid"] = guid
         dict["id"] = b.id
+        iconuri = getattr(b, 'icon_href')
+        if iconuri: dict["iconuri"] = iconuri
         index = getattr(b, 'index')
-        if index: dict["index"] = index
+        if index is not None: dict["index"] = index
         keyword = getattr(b, 'keyword')
         if keyword: dict["keyword"] = keyword
         dict["lastModified"] = convert_date_to_json(b.last_modified)
-        dict["parent"] = b.parent_idx
         dict["title"] = b.name.decode('utf-8')
         dict["type"] = "text/x-moz-place"
         dict["uri"] = uri = b.href
-        if uri.startswith('place:'):
-            if uri.startswith('place:folder'):
-                value = 'Recently Bookmarked'
-            elif uri.startswith('place:sort=8') or \
-                    uri.startswith('place:redirectsMode'):
-                value = 'Most Visited'
-            elif uri.startswith('place:sort=14') or \
-                    uri.startswith('place:type=6'):
-                value = 'Recent Tags'
-            else:
-                raise ValueError('UNKNOWN place URI: %s' % uri)
-            dict["annos"] = make_annos(value, name='Places/SmartBookmark')
-            del dict["dateAdded"]
-            del dict["lastModified"]
         self.folder_stack[-1].append(dict)
 
     def ruler(self, r, level):
@@ -93,9 +80,10 @@ class storage_json(Walker):
         if comment: dict["annos"] = make_annos(comment)
         dict["dateAdded"] = convert_date_to_json(r.add_date)
         dict["id"] = r.id
+        guid = getattr(r, 'guid')
+        if guid: dict["guid"] = guid
         dict["index"] = r.index
         dict["lastModified"] = convert_date_to_json(r.last_modified)
-        dict["parent"] = r.parent_idx
         dict["title"] = r.name.decode('utf-8')
         dict["type"] = "text/x-moz-place-separator"
         self.folder_stack[-1].append(dict)
@@ -127,7 +115,9 @@ class storage_json(Walker):
         self.current_folder = root_folder
 
         if "type" not in bookmarks_dict:
-           bookmarks_dict = bookmarks_dict["children"][0]
+           bookmarks_dict["id"] = "0"
+           bookmarks_dict["title"] = ""
+           bookmarks_dict["type"] = "text/x-moz-place-container"
         self.load_folder(root_folder, bookmarks_dict)
         if self.folder_stack:
             raise RuntimeError('Excessive folder stack: %s' % self.folder_stack)
@@ -139,8 +129,8 @@ class storage_json(Walker):
             raise ValueError("The object is not a Mozilla container")
 
         folder.id = fdict["id"]
+        folder.guid = fdict.get("guid")
         folder.index = fdict.get("index")
-        folder.parent_idx = fdict.get("parent")
         folder.root = fdict.get("root")
         folder.name = encode_title(fdict["title"])
 
@@ -150,6 +140,7 @@ class storage_json(Walker):
                     add_date=convert_date_from_json(record.get("dateAdded")),
                     comment=get_comment(record.get("annos")),
                     last_modified=convert_date_from_json(record.get("lastModified")))
+                folder.guid = record.get("guid")
                 self.current_folder.append(folder)
                 self.folder_stack.append(folder)
                 self.current_folder = folder
@@ -162,20 +153,21 @@ class storage_json(Walker):
                     last_modified=convert_date_from_json(record.get("lastModified")),
                     keyword=get_str(record, "keyword"),
                     comment=get_comment(record.get("annos")),
+                    icon_href=record.get("iconuri"),
                     charset=get_str(record, "charset"))
+                bookmark.guid = record.get("guid")
                 bookmark.id = record["id"]
                 bookmark.index = record.get("index")
-                bookmark.parent_idx = record.get("parent")
                 bookmark.name = encode_title(record["title"])
                 self.current_folder.append(bookmark)
 
             elif record["type"] == "text/x-moz-place-separator":
                 ruler = Ruler()
                 ruler.add_date = convert_date_from_json(record.get("dateAdded"))
+                ruler.guid = record.get("guid")
                 ruler.id = record["id"]
                 ruler.index = record["index"]
                 ruler.last_modified = convert_date_from_json(record.get("lastModified"))
-                ruler.parent_idx = record.get("parent")
                 ruler.name = encode_title(record["title"])
                 ruler.comment = get_comment(record.get("annos"))
                 self.current_folder.append(ruler)
@@ -192,14 +184,14 @@ class storage_json(Walker):
 
 def convert_date_to_json(date):
     if date:
-        date = int(date) * 10**6
+        date = int(date * 10**6)
     return date
 
 def convert_date_from_json(date):
     if date:
-        date = int(date)
+        date = float(date)
         if date > 10**10:
-            date /= 10**6
+            date /= 10.0**6
     return date
 
 def encode_title(title):
@@ -224,7 +216,5 @@ def make_annos(value, name="bookmarkProperties/description"):
     return [{
         "expires": 4,
         "flags": 0,
-        "mimeType": None,
         "name": name,
-        "type": 3,
         "value": value.decode('utf-8')}]

diff --git a/doc/ChangeLog b/doc/ChangeLog
index d6e6a7e0e358b86b9ebf434a10d7d4483095b58f..62879a353964ac3d025e76f57ae6fcfea0a47b9f 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -3,6 +3,8 @@ WHAT'S NEW in version 4.6.0
    Split simple robot: separate network operations and
    URL handling/HTML parsing.
 
+   Adapt JSON storage to recent Mozilla export format.
+
 
 WHAT'S NEW in version 4.5.7