X-Git-Url: https://git.phdru.name/?p=sqlconvert.git;a=blobdiff_plain;f=mysql2sql%2Fprocess_tokens.py;h=1e74ac9b69a5a416bb6cf8b0418219095b73b357;hp=7e323543f81f9baa5f65444e7edc8327e16d7eb7;hb=4c93c3d89685aba33fc45082022373eb93b6583e;hpb=7d3342fc8ae5d217736c9c055b53445c628da765

diff --git a/mysql2sql/process_tokens.py b/mysql2sql/process_tokens.py
index 7e32354..1e74ac9 100644
--- a/mysql2sql/process_tokens.py
+++ b/mysql2sql/process_tokens.py
@@ -33,16 +33,17 @@ if PY3:
 class StatementGrouper(object):
     """Collect lines and reparse until the last statement is complete"""
 
-    def __init__(self):
+    def __init__(self, encoding=None):
         self.lines = []
         self.statements = []
+        self.encoding = encoding
 
     def process_line(self, line):
         self.lines.append(line)
         self.process_lines()
 
     def process_lines(self):
-        statements = parse('\n'.join(self.lines))
+        statements = parse(''.join(self.lines), encoding=self.encoding)
         last_stmt = statements[-1]
         for i in xrange(len(last_stmt.tokens) - 1, 0, -1):
             token = last_stmt.tokens[i]
@@ -64,7 +65,7 @@ class StatementGrouper(object):
     def close(self):
         if not self.lines:
             return
-        tokens = parse('\n'.join(self.lines))
+        tokens = parse(''.join(self.lines), encoding=self.encoding)
         for token in tokens:
             if (token.ttype not in (Comment.Single, Comment.Multiline,
                                     Newline, Whitespace)):