Add MySQL-specific remove_directives() and process_statement()

diff --git a/ChangeLog b/ChangeLog
index e12e5ad86ad37c9565f0f7cfabee8b820cc31afb..1862c1fc875dfad149c29c1ff5d8683bc1d38713 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+Version 0.0.4 (2016-09-04)
+
+   Add MySQL-specific remove_directives() and process_statement().
+
 Version 0.0.3 (2016-09-04)
 
    Rename the project: mysql2py -> sqlconvert.

diff --git a/demo/group-file.py b/demo/group-file.py
index e59ff45dd8b707292d4c07bad6c0f348fa8759a9..b8a70214a2f475d41f8ef3907c53ea90275b782e 100755 (executable)
--- a/demo/group-file.py
+++ b/demo/group-file.py
@@ -3,7 +3,7 @@ from __future__ import print_function
 
 import sys
 from sqlconvert.print_tokens import print_tokens
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import process_statement
 from sqlconvert.process_tokens import find_error, StatementGrouper
 
 
@@ -17,7 +17,7 @@ def main(filename):
                     print("----------")
                     if find_error(statement):
                         print("ERRORS IN QUERY")
-                    requote_names(statement)
+                    process_statement(statement)
                     print_tokens(statement)
                     print()
                     statement._pprint_tree()

diff --git a/demo/group-sql.py b/demo/group-sql.py
index 854a0d135ec3124b0a6de9e0f06072b8132d2144..c0bf2d29a49c8c110ac25ea4537169362c4f37b8 100755 (executable)
--- a/demo/group-sql.py
+++ b/demo/group-sql.py
@@ -3,7 +3,7 @@ from __future__ import print_function
 
 import sys
 from sqlconvert.print_tokens import print_tokens
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import process_statement
 from sqlconvert.process_tokens import find_error, StatementGrouper
 
 
@@ -16,7 +16,7 @@ def main(*queries):
                 print("----------")
                 if find_error(statement):
                     print("ERRORS IN QUERY")
-                requote_names(statement)
+                process_statement(statement)
                 print_tokens(statement)
                 print()
                 statement._pprint_tree()

diff --git a/demo/parse-file.py b/demo/parse-file.py
index bf65eec3f61e620ce79aa1416ddd3d78eeaa3780..e19bde2f6b87dde3bca869dc35783285134c0d56 100755 (executable)
--- a/demo/parse-file.py
+++ b/demo/parse-file.py
@@ -4,7 +4,7 @@ from __future__ import print_function
 import sys
 from sqlparse import parse
 from sqlconvert.print_tokens import print_tokens
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import process_statement
 from sqlconvert.process_tokens import find_error
 
 
@@ -15,7 +15,7 @@ def main(filename):
                 print("----------")
                 if find_error(parsed):
                     print("ERRORS IN QUERY")
-                requote_names(parsed)
+                process_statement(parsed)
                 print_tokens(parsed)
                 print()
                 parsed._pprint_tree()

diff --git a/demo/parse-sql.py b/demo/parse-sql.py
index 1cbe0a11e8632f35c6a986db3a9acbe53f24b5d2..e4be365fc8eed012ea8cda8b490d91f4ff7651d8 100755 (executable)
--- a/demo/parse-sql.py
+++ b/demo/parse-sql.py
@@ -4,7 +4,7 @@ from __future__ import print_function
 import sys
 from sqlparse import parse
 from sqlconvert.print_tokens import print_tokens
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import process_statement
 from sqlconvert.process_tokens import find_error
 
 
@@ -14,7 +14,7 @@ def main(*queries):
             print("----------")
             if find_error(parsed):
                 print("ERRORS IN QUERY")
-            requote_names(parsed)
+            process_statement(parsed)
             print_tokens(parsed)
             print()
             parsed._pprint_tree()

diff --git a/scripts/mysql2sql b/scripts/mysql2sql
index 5ba8fd741e1f021e7133aa9f79393c108f161505..17ee67cd4fb123f08332e1fa37633f896bc66cc1 100755 (executable)
--- a/scripts/mysql2sql
+++ b/scripts/mysql2sql
@@ -8,7 +8,7 @@ import sys
 
 from sqlparse.compat import text_type
 from sqlconvert.print_tokens import print_tokens
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import process_statement
 from sqlconvert.process_tokens import StatementGrouper
 
 from m_lib.defenc import default_encoding
@@ -49,7 +49,7 @@ def main(infile, encoding, outfile, output_encoding, use_pbar):
         grouper.process_line(line)
         if grouper.statements:
             for statement in grouper.get_statements():
-                requote_names(statement)
+                process_statement(statement)
                 print_tokens(statement, outfile=outfile,
                              encoding=output_encoding)
     tokens = grouper.close()

diff --git a/sqlconvert/__version__.py b/sqlconvert/__version__.py
index ffcc925ae2ade5f8b863840f7c8a0e337341410c..156d6f9a28526eef969da384ac3b5b101f58f603 100644 (file)
--- a/sqlconvert/__version__.py
+++ b/sqlconvert/__version__.py
@@ -1 +1 @@
-__version__ = '0.0.3'
+__version__ = '0.0.4'

diff --git a/sqlconvert/process_mysql.py b/sqlconvert/process_mysql.py
index b11eccb38f4376f526a7b8b3130e618f7ff58c8f..5c63d0d1bc1de9c39c67455cd422ffc838a4220f 100644 (file)
--- a/sqlconvert/process_mysql.py
+++ b/sqlconvert/process_mysql.py
@@ -1,11 +1,12 @@
 
-from sqlparse.tokens import Name
+from sqlparse.sql import Comment
+from sqlparse import tokens as T
 
 
 def requote_names(token_list):
     """Remove backticks, quote non-lowercase identifiers"""
     for token in token_list.flatten():
-        if token.ttype is Name:
+        if token.ttype is T.Name:
             value = token.value
             if (value[0] == "`") and (value[-1] == "`"):
                 value = value[1:-1]
@@ -13,3 +14,23 @@ def requote_names(token_list):
                 token.normalized = token.value = value
             else:
                 token.normalized = token.value = '"%s"' % value
+
+
+def remove_directives(statement):
+    """Remove /*! directives */ from the first-level"""
+    new_tokens = []
+    for token in statement.tokens:
+        if isinstance(token, Comment):
+            subtokens = token.tokens
+            if subtokens:
+                comment = subtokens[0]
+                if comment.ttype is T.Comment.Multiline and \
+                        comment.value.startswith('/*!'):
+                    continue
+        new_tokens.append(token)
+    statement.tokens = new_tokens
+
+
+def process_statement(statement):
+    requote_names(statement)
+    remove_directives(statement)

diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index 692121ed321a84613e731b270ba8246997e9e587..3b0452a50cdcdc4c770b6ffbbfb653d29fc5286f 100755 (executable)
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -5,17 +5,12 @@ import unittest
 from sqlparse import parse
 
 from sqlconvert.print_tokens import tlist2str
-from sqlconvert.process_mysql import requote_names
+from sqlconvert.process_mysql import requote_names, remove_directives, \
+        process_statement
 from tests import main
 
 
 class TestTokens(unittest.TestCase):
-    def test_requote(self):
-        parsed = parse("select * from `T`")[0]
-        requote_names(parsed)
-        query = tlist2str(parsed)
-        self.assertEqual(query, 'SELECT * FROM "T"')
-
     def test_encoding(self):
         parsed = parse("insert into test (1, 'тест')", 'utf-8')[0]
         query = tlist2str(parsed).encode('utf-8')
@@ -27,6 +22,24 @@ class TestTokens(unittest.TestCase):
         query = tlist2str(parsed)
         self.assertEqual(query, u"INSERT INTO test (1, 'тест')")
 
+    def test_requote(self):
+        parsed = parse("select * from `T`")[0]
+        requote_names(parsed)
+        query = tlist2str(parsed)
+        self.assertEqual(query, 'SELECT * FROM "T"')
+
+    def test_directives(self):
+        parsed = parse("select /*! test */ * from /* test */ `T`")[0]
+        remove_directives(parsed)
+        query = tlist2str(parsed)
+        self.assertEqual(query, 'SELECT * FROM /* test */ `T`')
+
+    def test_process(self):
+        parsed = parse("select /*! test */ * from /* test */ `T`")[0]
+        process_statement(parsed)
+        query = tlist2str(parsed)
+        self.assertEqual(query, 'SELECT * FROM /* test */ "T"')
+
 
 if __name__ == "__main__":
     main()