From b83ec9eaf7e388f4061c5e13dd930fd108e4bd77 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Sat, 10 Sep 2016 20:35:02 +0300 Subject: [PATCH] Unescape strings --- ChangeLog | 2 ++ README.txt | 3 ++- docs/mysql2sql.rst | 5 +++-- sqlconvert/process_mysql.py | 15 +++++++++++++++ tests/test_tokens.py | 27 +++++++++++++++++---------- 5 files changed, 39 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6d0e194..ec6b82f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ Version 0.0.6 (2016-09-??) Rename remove_directives -> remove_directive_tokens. + Unescape strings. + Version 0.0.5 (2016-09-07) Remove /*! directives */; and newlines after them. diff --git a/README.txt b/README.txt index 86b1b05..2e44474 100644 --- a/README.txt +++ b/README.txt @@ -5,7 +5,8 @@ License: GPL This is sqlconvert, a a library to perform SQL convertions. It uses sqlparse to parse SQL. -The library is in its initial phase and currently cannot do much. +The library is in the early stage of development and currently cannot do +much. The first goal is to implemet mysq2sql, a script intended primarily to convert mysqldump output (especially with extended INSERT syntax) to diff --git a/docs/mysql2sql.rst b/docs/mysql2sql.rst index 9c1892d..12e574b 100644 --- a/docs/mysql2sql.rst +++ b/docs/mysql2sql.rst @@ -5,8 +5,9 @@ This is mysql2sql, a mysql to sql converter. It is primary intended to convert mysqldump (especially with extended INSERT syntax) to standard SQL to load at least to PostgreSQL or SQLite. -The program is in its initial phase and currently cannot do much. It only -removes /\*! directives \*/ and passes everything else unmodified. +The program is in the early stage of development and currently cannot do much. +It removes /\*! directives \*/, unescapes strings and passes everything else +unmodified. .. highlight:: none diff --git a/sqlconvert/process_mysql.py b/sqlconvert/process_mysql.py index f91f516..d72479e 100644 --- a/sqlconvert/process_mysql.py +++ b/sqlconvert/process_mysql.py @@ -49,6 +49,21 @@ def requote_names(token_list): token.normalized = token.value = '"%s"' % value +def unescape_strings(token_list): + """Unescape strings""" + for token in token_list.flatten(): + if token.ttype is T.String.Single: + value = token.value + for orig, repl in ( + ('\\"', '"'), + ("\\'", "''"), + ('\\\032', '\032'), + ): + value = value.replace(orig, repl) + token.normalized = token.value = value + + def process_statement(statement): remove_directive_tokens(statement) requote_names(statement) + unescape_strings(statement) diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 2486654..de11485 100755 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -5,8 +5,9 @@ import unittest from sqlparse import parse from sqlconvert.print_tokens import tlist2str -from sqlconvert.process_mysql import remove_directive_tokens, requote_names, \ - is_directive_statement, process_statement +from sqlconvert.process_mysql import remove_directive_tokens, \ + is_directive_statement, requote_names, unescape_strings, \ + process_statement from tests import main @@ -22,17 +23,11 @@ class TestTokens(unittest.TestCase): query = tlist2str(parsed) self.assertEqual(query, u"INSERT INTO test (1, 'тест')") - def test_requote(self): - parsed = parse("select * from `T`")[0] - requote_names(parsed) - query = tlist2str(parsed) - self.assertEqual(query, 'SELECT * FROM "T"') - def test_directive(self): parsed = parse("select /*! test */ * from /* test */ `T`")[0] remove_directive_tokens(parsed) query = tlist2str(parsed) - self.assertEqual(query, 'SELECT * FROM /* test */ `T`') + self.assertEqual(query, u'SELECT * FROM /* test */ `T`') def test_directive_statement(self): parsed = parse("/*! test */ test ;")[0] @@ -40,11 +35,23 @@ class TestTokens(unittest.TestCase): parsed = parse("/*! test */ ;")[0] self.assertTrue(is_directive_statement(parsed)) + def test_requote(self): + parsed = parse("select * from `T`")[0] + requote_names(parsed) + query = tlist2str(parsed) + self.assertEqual(query, u'SELECT * FROM "T"') + + def test_string(self): + parsed = parse("insert into test values ('\"test\\\"')")[0] + unescape_strings(parsed) + query = tlist2str(parsed) + self.assertEqual(query, u"INSERT INTO test VALUES ('\"test\"')") + def test_process(self): parsed = parse("select /*! test */ * from /* test */ `T`")[0] process_statement(parsed) query = tlist2str(parsed) - self.assertEqual(query, 'SELECT * FROM /* test */ "T"') + self.assertEqual(query, u'SELECT * FROM /* test */ "T"') if __name__ == "__main__": -- 2.39.2