Process input stream line by line

[sqlconvert.git] / scripts / group-file.py

diff --git a/scripts/group-file.py b/scripts/group-file.py
index 41f9b3318b47248100e892db2d4362201961594f..92f8fb9100d8839d7f0150e899db8c5886d231d8 100755 (executable)
--- a/scripts/group-file.py
+++ b/scripts/group-file.py
@@ -2,7 +2,6 @@
 from __future__ import print_function
 
 import sys
-from sqlparse import parse
 from mysql2sql.print_tokens import print_tokens
 from mysql2sql.process_tokens import requote_names, find_error, \
     StatementGrouper
@@ -12,7 +11,7 @@ def main(filename):
     grouper = StatementGrouper()
     with open(filename) as infile:
         for line in infile:
-            grouper.process(parse(line)[0])
+            grouper.process_line(line)
             if grouper.statements:
                 for statement in grouper.get_statements():
                     print("----------")
@@ -23,7 +22,10 @@ def main(filename):
                     print()
                     statement._pprint_tree()
                 print("----------")
-    grouper.close()
+    tokens = grouper.close()
+    for token in tokens:
+        print_tokens(token)
+        print(repr(token))
 
 
 if __name__ == '__main__':