def print_tokens(token_list, outfile=sys.stdout, encoding=None):
if encoding:
- outfile = getattr(outfile, 'buffer', outfile)
+ buffer = getattr(outfile, 'buffer', outfile)
+ else:
+ buffer = outfile
for token in token_list.flatten():
normalized = token.normalized
if encoding:
normalized = normalized.encode(encoding)
- outfile.write(normalized)
+ buffer.write(normalized)
+ if buffer is not outfile:
+ buffer.flush()
+ outfile.flush()
def tlist2str(token_list):
- return u''.join(token.normalized for token in token_list.flatten())
+ return u''.join(
+ token.normalized for token in token_list.flatten()
+ ).replace(' ', ' ')