import sys
-def print_tokens(token_list, outfile=sys.stdout):
+def print_tokens(token_list, outfile=sys.stdout, encoding=None):
+ if encoding:
+ outfile = getattr(outfile, 'buffer', outfile)
for token in token_list.flatten():
- outfile.write(token.normalized)
+ normalized = token.normalized
+ if encoding:
+ normalized = normalized.encode(encoding)
+ outfile.write(normalized)
def tlist2str(token_list):
- return ''.join(token.normalized for token in token_list.flatten())
+ return u''.join(token.normalized for token in token_list.flatten())