sqlconvert/print_tokens.py

   1
   2 import sys
   3
   4
   5 def print_tokens(token_list, outfile=sys.stdout, encoding=None):
   6     if encoding:
   7         buffer = getattr(outfile, 'buffer', outfile)
   8     else:
   9         buffer = outfile
  10     for token in token_list.flatten():
  11         normalized = token.normalized
  12         if encoding:
  13             normalized = normalized.encode(encoding)
  14         buffer.write(normalized)
  15     if buffer is not outfile:
  16         buffer.flush()
  17     outfile.flush()
  18
  19
  20 def tlist2str(token_list):
  21     return u''.join(token.normalized for token in token_list.flatten())