Storage/bkmk_stjson.py

   1 """Bookmarks storage manager - json
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4 """
   5
   6 __author__ = "Oleg Broytman <phd@phdru.name>"
   7 __copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design"
   8 __license__ = "GNU GPL"
   9
  10 __all__ = ['storage_json']
  11
  12
  13 try:
  14    import json
  15 except ImportError:
  16    import simplejson as json
  17
  18 from bkmk_objects import Folder, Bookmark, Ruler, Walker
  19
  20
  21 class storage_json(Walker):
  22     filename = "bookmarks_db.json"
  23
  24     def root_folder(self, f):
  25         self.dict = dict = {}
  26         dict["children"] = children = []
  27         self.folder_stack = [children]
  28         dict["dateAdded"] = convert_date_to_json(f.add_date)
  29         dict["id"] = f.id
  30         dict["lastModified"] = convert_date_to_json(f.last_modified)
  31         dict["root"] = "placesRoot"
  32         dict["title"] = ""
  33         dict["type"] = "text/x-moz-place-container"
  34
  35     def start_folder(self, f, level):
  36         dict = {}
  37         comment = getattr(f, 'comment')
  38         if comment: dict["annos"] = make_annos(comment)
  39         dict["children"] = children = []
  40         dict["dateAdded"] = convert_date_to_json(f.add_date)
  41         dict["id"] = f.id
  42         index = getattr(f, 'index')
  43         if index: dict["index"] = index
  44         dict["lastModified"] = convert_date_to_json(f.last_modified)
  45         parent_idx = getattr(f, 'parent_idx')
  46         if parent_idx: dict["parent"] = parent_idx
  47         root = getattr(f, 'root')
  48         if root: dict["root"] = root
  49         dict["title"] = f.name.decode('utf-8')
  50         dict["type"] = "text/x-moz-place-container"
  51         self.folder_stack[-1].append(dict)
  52         self.folder_stack.append(children)
  53
  54     def end_folder(self, f, level):
  55         del self.folder_stack[-1]
  56
  57     def bookmark(self, b, level):
  58         dict = {}
  59         comment = getattr(b, 'comment')
  60         if comment: dict["annos"] = make_annos(comment)
  61         charset = getattr(b, 'charset')
  62         if charset: dict["charset"] = charset
  63         dict["dateAdded"] = convert_date_to_json(b.add_date)
  64         dict["id"] = b.id
  65         index = getattr(b, 'index')
  66         if index: dict["index"] = index
  67         keyword = getattr(b, 'keyword')
  68         if keyword: dict["keyword"] = keyword
  69         dict["lastModified"] = convert_date_to_json(b.last_modified)
  70         dict["parent"] = b.parent_idx
  71         dict["title"] = b.name.decode('utf-8')
  72         dict["type"] = "text/x-moz-place"
  73         dict["uri"] = uri = b.href
  74         if uri.startswith('place:'):
  75             if uri.startswith('place:folder'):
  76                 value = 'Recently Bookmarked'
  77             elif uri.startswith('place:sort=8') or \
  78                     uri.startswith('place:redirectsMode'):
  79                 value = 'Most Visited'
  80             elif uri.startswith('place:sort=14') or \
  81                     uri.startswith('place:type=6'):
  82                 value = 'Recent Tags'
  83             else:
  84                 raise ValueError('UNKNOWN place URI: %s' % uri)
  85             dict["annos"] = make_annos(value, name='Places/SmartBookmark')
  86             del dict["dateAdded"]
  87             del dict["lastModified"]
  88         self.folder_stack[-1].append(dict)
  89
  90     def ruler(self, r, level):
  91         dict = {}
  92         comment = getattr(r, 'comment')
  93         if comment: dict["annos"] = make_annos(comment)
  94         dict["dateAdded"] = convert_date_to_json(r.add_date)
  95         dict["id"] = r.id
  96         dict["index"] = r.index
  97         dict["lastModified"] = convert_date_to_json(r.last_modified)
  98         dict["parent"] = r.parent_idx
  99         dict["title"] = r.name.decode('utf-8')
 100         dict["type"] = "text/x-moz-place-separator"
 101         self.folder_stack[-1].append(dict)
 102
 103     def store(self, root_folder):
 104         root_folder.walk_depth(self)
 105
 106         outfile = open(self.filename, 'wb')
 107         json.dump(self.dict, outfile)
 108         outfile.close()
 109         del self.dict
 110
 111     def load(self):
 112         infile = open(self.filename, 'rb')
 113         bkmk_s = infile.read()
 114         infile.close()
 115
 116         # Work around a bug in Mozilla - remove the trailing comma
 117         bkmk_s = bkmk_s.strip().replace(',]', ']')
 118         bookmarks_dict = json.loads(bkmk_s)
 119         del bkmk_s
 120
 121         root_folder = Folder()
 122         root_folder.header = ''
 123         root_folder.add_date = convert_date_from_json(bookmarks_dict.get("dateAdded"))
 124         root_folder.comment = ''
 125         root_folder.last_modified = convert_date_from_json(bookmarks_dict.get("lastModified"))
 126         self.folder_stack = [root_folder]
 127         self.current_folder = root_folder
 128
 129         if "type" not in bookmarks_dict:
 130            bookmarks_dict = bookmarks_dict["children"][0]
 131         self.load_folder(root_folder, bookmarks_dict)
 132         if self.folder_stack:
 133             raise RuntimeError('Excessive folder stack: %s' % self.folder_stack)
 134
 135         return root_folder
 136
 137     def load_folder(self, folder, fdict):
 138         if fdict["type"] != "text/x-moz-place-container":
 139             raise ValueError("The object is not a Mozilla container")
 140
 141         folder.id = fdict["id"]
 142         folder.index = fdict.get("index")
 143         folder.parent_idx = fdict.get("parent")
 144         folder.root = fdict.get("root")
 145         folder.name = encode_title(fdict["title"])
 146
 147         for record in fdict["children"]:
 148             if record["type"] == "text/x-moz-place-container":
 149                 folder = Folder(
 150                     add_date=convert_date_from_json(record.get("dateAdded")),
 151                     comment=get_comment(record.get("annos")),
 152                     last_modified=convert_date_from_json(record.get("lastModified")))
 153                 self.current_folder.append(folder)
 154                 self.folder_stack.append(folder)
 155                 self.current_folder = folder
 156                 self.load_folder(folder, record)
 157
 158             elif record["type"] == "text/x-moz-place":
 159                 bookmark = Bookmark(
 160                     href=record["uri"].encode('utf-8'),
 161                     add_date=convert_date_from_json(record.get("dateAdded")),
 162                     last_modified=convert_date_from_json(record.get("lastModified")),
 163                     keyword=get_str(record, "keyword"),
 164                     comment=get_comment(record.get("annos")),
 165                     charset=get_str(record, "charset"))
 166                 bookmark.id = record["id"]
 167                 bookmark.index = record.get("index")
 168                 bookmark.parent_idx = record.get("parent")
 169                 bookmark.name = encode_title(record["title"])
 170                 self.current_folder.append(bookmark)
 171
 172             elif record["type"] == "text/x-moz-place-separator":
 173                 ruler = Ruler()
 174                 ruler.add_date = convert_date_from_json(record.get("dateAdded"))
 175                 ruler.id = record["id"]
 176                 ruler.index = record["index"]
 177                 ruler.last_modified = convert_date_from_json(record.get("lastModified"))
 178                 ruler.parent_idx = record.get("parent")
 179                 ruler.name = encode_title(record["title"])
 180                 ruler.comment = get_comment(record.get("annos"))
 181                 self.current_folder.append(ruler)
 182
 183             else:
 184                 raise ValueError('Unknown record type "%s"' % record["type"])
 185
 186         del self.folder_stack[-1]
 187         if self.folder_stack:
 188             self.current_folder = self.folder_stack[-1]
 189         else:
 190             self.current_folder = None
 191
 192
 193 def convert_date_to_json(date):
 194     if date:
 195         date = int(date) * 10**6
 196     return date
 197
 198 def convert_date_from_json(date):
 199     if date:
 200         date = int(date)
 201         if date > 10**10:
 202             date /= 10**6
 203     return date
 204
 205 def encode_title(title):
 206     return title.encode("UTF-8", "xmlcharrefreplace")
 207
 208 def get_str(record, name):
 209     if name in record:
 210         return record[name].encode('utf-8')
 211     return ''
 212
 213 def get_comment(annos):
 214     if not annos:
 215         return ''
 216
 217     for a in annos:
 218         if a["name"] == "bookmarkProperties/description":
 219             return a["value"].encode('utf-8')
 220
 221     return ''
 222
 223 def make_annos(value, name="bookmarkProperties/description"):
 224     return [{
 225         "expires": 4,
 226         "flags": 0,
 227         "mimeType": None,
 228         "name": name,
 229         "type": 3,
 230         "value": value.decode('utf-8')}]