Storage/bkmk_stjson.py

   1 """Bookmarks storage manager - json
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4 """
   5
   6 __version__ = "$Revision$"[11:-2]
   7 __revision__ = "$Id$"[5:-2]
   8 __date__ = "$Date$"[7:-2]
   9 __author__ = "Oleg Broytman <phd@phdru.name>"
  10 __copyright__ = "Copyright (C) 2010, 2011 PhiloSoft Design"
  11 __license__ = "GNU GPL"
  12
  13 __all__ = ['storage_json']
  14
  15
  16 try:
  17    import json
  18 except ImportError:
  19    import simplejson as json
  20
  21 from bkmk_objects import Folder, Bookmark, Ruler, Walker
  22
  23
  24 class storage_json(Walker):
  25     filename = "bookmarks_db.json"
  26
  27     def root_folder(self, f):
  28         self.dict = dict = {}
  29         dict["children"] = children = []
  30         self.folder_stack = [children]
  31         dict["dateAdded"] = convert_date_to_json(f.add_date)
  32         dict["id"] = f.id
  33         dict["lastModified"] = convert_date_to_json(f.last_modified)
  34         dict["root"] = "placesRoot"
  35         dict["title"] = ""
  36         dict["type"] = "text/x-moz-place-container"
  37
  38     def start_folder(self, f, level):
  39         dict = {}
  40         comment = getattr(f, 'comment')
  41         if comment: dict["annos"] = make_annos(comment)
  42         dict["children"] = children = []
  43         dict["dateAdded"] = convert_date_to_json(f.add_date)
  44         dict["id"] = f.id
  45         index = getattr(f, 'index')
  46         if index: dict["index"] = index
  47         dict["lastModified"] = convert_date_to_json(f.last_modified)
  48         parent_idx = getattr(f, 'parent_idx')
  49         if parent_idx: dict["parent"] = parent_idx
  50         root = getattr(f, 'root')
  51         if root: dict["root"] = root
  52         dict["title"] = f.name.decode('utf-8')
  53         dict["type"] = "text/x-moz-place-container"
  54         self.folder_stack[-1].append(dict)
  55         self.folder_stack.append(children)
  56
  57     def end_folder(self, f, level):
  58         del self.folder_stack[-1]
  59
  60     def bookmark(self, b, level):
  61         dict = {}
  62         comment = getattr(b, 'comment')
  63         if comment: dict["annos"] = make_annos(comment)
  64         charset = getattr(b, 'charset')
  65         if charset: dict["charset"] = charset
  66         dict["dateAdded"] = convert_date_to_json(b.add_date)
  67         dict["id"] = b.id
  68         index = getattr(b, 'index')
  69         if index: dict["index"] = index
  70         keyword = getattr(b, 'keyword')
  71         if keyword: dict["keyword"] = keyword
  72         dict["lastModified"] = convert_date_to_json(b.last_modified)
  73         dict["parent"] = b.parent_idx
  74         dict["title"] = b.name.decode('utf-8')
  75         dict["type"] = "text/x-moz-place"
  76         dict["uri"] = uri = b.href
  77         if uri.startswith('place:'):
  78             if uri.startswith('place:sort=8') \
  79                     or uri.startswith('place://sort=8'):
  80                 value = 'MostVisited'
  81             elif uri.startswith('place://redirectsMode'):
  82                 value = 'Most Visited'
  83             elif uri.startswith('place:folder=BOOKMARKS_MENU') \
  84                     or uri.startswith('place://folder=BOOKMARKS_MENU'):
  85                 value = 'RecentlyBookmarked'
  86             elif uri.startswith('place://folder'):
  87                 value = 'Recently Bookmarked'
  88             elif uri.startswith('place:sort=14') \
  89                     or uri.startswith('place://sort=14'):
  90                 value = 'RecentTags'
  91             elif uri.startswith('place://type'):
  92                 value = 'Recent Tags'
  93             else:
  94                 raise ValueError('UNKNOWN place URI: %s' % uri)
  95             dict["annos"] = make_annos(value, name='Places/SmartBookmark')
  96             del dict["dateAdded"]
  97             del dict["lastModified"]
  98         self.folder_stack[-1].append(dict)
  99
 100     def ruler(self, r, level):
 101         dict = {}
 102         comment = getattr(r, 'comment')
 103         if comment: dict["annos"] = make_annos(comment)
 104         dict["dateAdded"] = convert_date_to_json(r.add_date)
 105         dict["id"] = r.id
 106         dict["index"] = r.index
 107         dict["lastModified"] = convert_date_to_json(r.last_modified)
 108         dict["parent"] = r.parent_idx
 109         dict["title"] = r.name.decode('utf-8')
 110         dict["type"] = "text/x-moz-place-separator"
 111         self.folder_stack[-1].append(dict)
 112
 113     def store(self, root_folder):
 114         root_folder.walk_depth(self)
 115
 116         outfile = open(self.filename, 'wb')
 117         json.dump(self.dict, outfile)
 118         outfile.close()
 119         del self.dict
 120
 121     def load(self):
 122         infile = open(self.filename, 'rb')
 123         bkmk_s = infile.read()
 124         infile.close()
 125
 126         # Work around a bug in Mozilla - remove the trailing comma
 127         bkmk_s = bkmk_s.strip().replace(',]', ']')
 128         bookmarks_dict = json.loads(bkmk_s)
 129         del bkmk_s
 130
 131         root_folder = Folder()
 132         root_folder.header = ''
 133         root_folder.add_date = convert_date_from_json(bookmarks_dict["dateAdded"])
 134         root_folder.comment = ''
 135         root_folder.last_modified = convert_date_from_json(bookmarks_dict["lastModified"])
 136         self.folder_stack = [root_folder]
 137         self.current_folder = root_folder
 138
 139         self.load_folder(root_folder, bookmarks_dict)
 140         if self.folder_stack:
 141             raise RuntimeError('Excessive folder stack: %s' % self.folder_stack)
 142
 143         return root_folder
 144
 145     def load_folder(self, folder, fdict):
 146         if fdict["type"] != "text/x-moz-place-container":
 147             raise ValueError("The object is not a Mozilla container")
 148
 149         folder.id = fdict["id"]
 150         folder.index = fdict.get("index")
 151         folder.parent_idx = fdict.get("parent")
 152         folder.root = fdict.get("root")
 153         folder.name = encode_title(fdict["title"])
 154
 155         for record in fdict["children"]:
 156             if record["type"] == "text/x-moz-place-container":
 157                 folder = Folder(
 158                     add_date=convert_date_from_json(record["dateAdded"]),
 159                     comment=get_comment(record.get("annos")),
 160                     last_modified=convert_date_from_json(record["lastModified"]))
 161                 self.current_folder.append(folder)
 162                 self.folder_stack.append(folder)
 163                 self.current_folder = folder
 164                 self.load_folder(folder, record)
 165
 166             elif record["type"] == "text/x-moz-place":
 167                 bookmark = Bookmark(
 168                     href=record["uri"].encode('utf-8'),
 169                     add_date=convert_date_from_json(record.get("dateAdded")),
 170                     last_modified=convert_date_from_json(record.get("lastModified")),
 171                     keyword=get_str(record, "keyword"),
 172                     comment=get_comment(record.get("annos")),
 173                     charset=get_str(record, "charset"))
 174                 bookmark.id = record["id"]
 175                 bookmark.index = record.get("index")
 176                 bookmark.parent_idx = record["parent"]
 177                 bookmark.name = encode_title(record["title"])
 178                 self.current_folder.append(bookmark)
 179
 180             elif record["type"] == "text/x-moz-place-separator":
 181                 ruler = Ruler()
 182                 ruler.add_date = convert_date_from_json(record["dateAdded"])
 183                 ruler.id = record["id"]
 184                 ruler.index = record["index"]
 185                 ruler.last_modified = convert_date_from_json(record["lastModified"])
 186                 ruler.parent_idx = record["parent"]
 187                 ruler.name = encode_title(record["title"])
 188                 ruler.comment = get_comment(record.get("annos"))
 189                 self.current_folder.append(ruler)
 190
 191             else:
 192                 raise ValueError('Unknown record type "%s"' % record["type"])
 193
 194         del self.folder_stack[-1]
 195         if self.folder_stack:
 196             self.current_folder = self.folder_stack[-1]
 197         else:
 198             self.current_folder = None
 199
 200
 201 def convert_date_to_json(date):
 202     if date:
 203         date = int(date) * 10**6
 204     return date
 205
 206 def convert_date_from_json(date):
 207     if date:
 208         date = int(date)
 209         if date > 10**10:
 210             date /= 10**6
 211     return date
 212
 213 def encode_title(title):
 214     return title.encode("UTF-8", "xmlcharrefreplace")
 215
 216 def get_str(record, name):
 217     if name in record:
 218         return record[name].encode('utf-8')
 219     return ''
 220
 221 def get_comment(annos):
 222     if not annos:
 223         return ''
 224
 225     for a in annos:
 226         if a["name"] == "bookmarkProperties/description" and \
 227                 a["type"] == 3:
 228             return a["value"].encode('utf-8')
 229
 230     return ''
 231
 232 def make_annos(value, name="bookmarkProperties/description"):
 233     return [{
 234         "expires": 4,
 235         "flags": 0,
 236         "mimeType": None,
 237         "name": name,
 238         "type": 3,
 239         "value": value.decode('utf-8')}]