5 def print_tokens(token_list, outfile=sys.stdout, encoding=None):
7 buffer = getattr(outfile, 'buffer', outfile)
10 for token in token_list.flatten():
11 normalized = token.normalized
13 normalized = normalized.encode(encoding)
14 buffer.write(normalized)
15 if buffer is not outfile:
20 def tlist2str(token_list):
21 return u''.join(token.normalized for token in token_list.flatten())